[ 475.330854] env[62736]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=62736) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 475.331201] env[62736]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=62736) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 475.331316] env[62736]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=62736) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 475.331680] env[62736]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 475.418011] env[62736]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=62736) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 475.426763] env[62736]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.009s {{(pid=62736) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 475.469122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Creating reply queue: reply_c579f1f3113446edbaca3803c2d26d58 [ 475.477895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Expecting reply to msg 6e2c9b8e8c9140f3bdf331cfae5e4f43 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 475.491595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e2c9b8e8c9140f3bdf331cfae5e4f43 [ 476.026964] env[62736]: INFO nova.virt.driver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 476.097352] env[62736]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 476.097562] env[62736]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 476.097625] env[62736]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=62736) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 479.183939] env[62736]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-f332398b-7b53-4cc6-8d23-64616716e468 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.200755] env[62736]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=62736) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 479.200933] env[62736]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-c44838d6-f4a6-4478-9c95-feb30601e8f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.232650] env[62736]: INFO oslo_vmware.api [-] Successfully established new session; session ID is 0da9a. [ 479.232800] env[62736]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.135s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 479.233381] env[62736]: INFO nova.virt.vmwareapi.driver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] VMware vCenter version: 7.0.3 [ 479.236779] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b5a676a-27d0-497b-9e02-9fe792b9f555 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.254263] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49deb576-044e-4397-ac06-f54bcf378b46 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.260174] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fc494f1-5222-436b-b1a4-e988a1888777 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.266760] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cec2af0-ef66-416b-b8db-9c5a7d989f0f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.279422] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bb30049-57ed-44aa-8eac-9acba6c7ddd2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.285225] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbbf2350-7538-4c86-a7f4-1f60c5e0772a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.315444] env[62736]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-a4849c4e-7c70-4f87-92ad-08f51cce7022 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 479.320823] env[62736]: DEBUG nova.virt.vmwareapi.driver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Extension org.openstack.compute already exists. {{(pid=62736) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:224}} [ 479.323495] env[62736]: INFO nova.compute.provider_config [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 479.324172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Expecting reply to msg d1358aa4481344ad8112e78cf591baaa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 479.341046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1358aa4481344ad8112e78cf591baaa [ 479.826762] env[62736]: DEBUG nova.context [None req-9f80eb66-6a9d-4b8c-b8ed-0f7a9ecd17c8 None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),2d4e634d-6292-45c4-9eb1-09b106cacc48(cell1) {{(pid=62736) load_cells /opt/stack/nova/nova/context.py:464}} [ 479.828881] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 479.829104] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 479.829862] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 479.830335] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Acquiring lock "2d4e634d-6292-45c4-9eb1-09b106cacc48" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 479.830530] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Lock "2d4e634d-6292-45c4-9eb1-09b106cacc48" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 479.831536] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Lock "2d4e634d-6292-45c4-9eb1-09b106cacc48" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 479.851825] env[62736]: INFO dbcounter [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Registered counter for database nova_cell0 [ 479.860073] env[62736]: INFO dbcounter [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Registered counter for database nova_cell1 [ 479.863270] env[62736]: DEBUG oslo_db.sqlalchemy.engines [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62736) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 479.863615] env[62736]: DEBUG oslo_db.sqlalchemy.engines [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=62736) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 479.868387] env[62736]: ERROR nova.db.main.api [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 479.868387] env[62736]: result = function(*args, **kwargs) [ 479.868387] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 479.868387] env[62736]: return func(*args, **kwargs) [ 479.868387] env[62736]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 479.868387] env[62736]: result = fn(*args, **kwargs) [ 479.868387] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 479.868387] env[62736]: return f(*args, **kwargs) [ 479.868387] env[62736]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 479.868387] env[62736]: return db.service_get_minimum_version(context, binaries) [ 479.868387] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 479.868387] env[62736]: _check_db_access() [ 479.868387] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 479.868387] env[62736]: stacktrace = ''.join(traceback.format_stack()) [ 479.868387] env[62736]: [ 479.869381] env[62736]: ERROR nova.db.main.api [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 479.869381] env[62736]: result = function(*args, **kwargs) [ 479.869381] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 479.869381] env[62736]: return func(*args, **kwargs) [ 479.869381] env[62736]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 479.869381] env[62736]: result = fn(*args, **kwargs) [ 479.869381] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 479.869381] env[62736]: return f(*args, **kwargs) [ 479.869381] env[62736]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 479.869381] env[62736]: return db.service_get_minimum_version(context, binaries) [ 479.869381] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 479.869381] env[62736]: _check_db_access() [ 479.869381] env[62736]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 479.869381] env[62736]: stacktrace = ''.join(traceback.format_stack()) [ 479.869381] env[62736]: [ 479.870099] env[62736]: WARNING nova.objects.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 479.870099] env[62736]: WARNING nova.objects.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Failed to get minimum service version for cell 2d4e634d-6292-45c4-9eb1-09b106cacc48 [ 479.870322] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Acquiring lock "singleton_lock" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 479.870482] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Acquired lock "singleton_lock" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 479.870724] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Releasing lock "singleton_lock" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 479.871052] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Full set of CONF: {{(pid=62736) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 479.871197] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ******************************************************************************** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 479.871325] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] Configuration options gathered from: {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 479.871461] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 479.871670] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 479.871800] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ================================================================================ {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 479.872030] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] allow_resize_to_same_host = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.872243] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] arq_binding_timeout = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.872379] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] backdoor_port = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.872505] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] backdoor_socket = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.872670] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] block_device_allocate_retries = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.872834] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] block_device_allocate_retries_interval = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873006] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cert = self.pem {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873174] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873341] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute_monitors = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873507] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] config_dir = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873675] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] config_drive_format = iso9660 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873809] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.873974] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] config_source = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874139] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] console_host = devstack {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874304] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] control_exchange = nova {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874464] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cpu_allocation_ratio = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874623] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] daemon = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] debug = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.874942] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_access_ip_network_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.875104] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_availability_zone = nova {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.875259] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_ephemeral_format = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.875424] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_green_pool_size = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.875660] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.875898] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] default_schedule_zone = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876028] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] disk_allocation_ratio = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876152] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] enable_new_services = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876329] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] enabled_apis = ['osapi_compute'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876495] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] enabled_ssl_apis = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876656] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] flat_injected = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876819] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] force_config_drive = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.876980] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] force_raw_images = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.877150] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] graceful_shutdown_timeout = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.877312] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] heal_instance_info_cache_interval = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.877529] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] host = cpu-1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.877698] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] initial_cpu_allocation_ratio = 4.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.877916] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] initial_disk_allocation_ratio = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.878107] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] initial_ram_allocation_ratio = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.878338] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.878507] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_build_timeout = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.878670] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_delete_interval = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.878835] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_format = [instance: %(uuid)s] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879000] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_name_template = instance-%08x {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879160] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_usage_audit = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879364] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_usage_audit_period = month {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879539] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879708] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] instances_path = /opt/stack/data/nova/instances {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.879864] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] internal_service_availability_zone = internal {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880030] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] key = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880202] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] live_migration_retry_count = 30 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880374] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_color = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880539] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_config_append = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880706] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.880864] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_dir = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881023] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881151] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_options = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881314] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_rotate_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881485] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_rotate_interval_type = days {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881681] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] log_rotation_type = none {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881816] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.881944] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882113] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882278] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882409] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882572] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] long_rpc_timeout = 1800 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882734] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_concurrent_builds = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.882892] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_concurrent_live_migrations = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883051] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_concurrent_snapshots = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883210] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_local_block_devices = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883369] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_logfile_count = 30 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883531] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] max_logfile_size_mb = 200 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883692] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] maximum_instance_delete_attempts = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.883859] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metadata_listen = 0.0.0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884036] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metadata_listen_port = 8775 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884213] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metadata_workers = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884377] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] migrate_max_retries = -1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884545] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] mkisofs_cmd = genisoimage {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884751] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] my_block_storage_ip = 10.180.1.21 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.884885] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] my_ip = 10.180.1.21 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885051] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] network_allocate_retries = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885229] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885399] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] osapi_compute_listen = 0.0.0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885562] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] osapi_compute_listen_port = 8774 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885728] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] osapi_compute_unique_server_name_scope = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.885895] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] osapi_compute_workers = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886059] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] password_length = 12 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886221] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] periodic_enable = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886589] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] periodic_fuzzy_delay = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886589] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] pointer_model = usbtablet {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886709] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] preallocate_images = none {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886843] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] publish_errors = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.886972] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] pybasedir = /opt/stack/nova {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.887127] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ram_allocation_ratio = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.887287] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rate_limit_burst = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.887455] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rate_limit_except_level = CRITICAL {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.887614] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rate_limit_interval = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888014] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reboot_timeout = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888014] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reclaim_instance_interval = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888135] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] record = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888247] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reimage_timeout_per_gb = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888411] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] report_interval = 120 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888571] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rescue_timeout = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888729] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reserved_host_cpus = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.888887] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reserved_host_disk_mb = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889048] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reserved_host_memory_mb = 512 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889207] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] reserved_huge_pages = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889398] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] resize_confirm_window = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889566] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] resize_fs_using_block_device = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889727] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] resume_guests_state_on_host_boot = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.889893] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.890054] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] rpc_response_timeout = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.890217] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] run_external_periodic_tasks = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.890458] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] running_deleted_instance_action = reap {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.890714] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] running_deleted_instance_poll_interval = 1800 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.890904] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] running_deleted_instance_timeout = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891071] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler_instance_sync_interval = 120 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891243] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_down_time = 720 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891416] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] servicegroup_driver = db {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891596] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] shelved_offload_time = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891775] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] shelved_poll_interval = 3600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.891946] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] shutdown_timeout = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.892127] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] source_is_ipv6 = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.892291] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ssl_only = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.892550] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.892720] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] sync_power_state_interval = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.892883] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] sync_power_state_pool_size = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893049] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] syslog_log_facility = LOG_USER {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893207] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] tempdir = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893366] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] timeout_nbd = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893533] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] transport_url = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893696] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] update_resources_interval = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.893856] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_cow_images = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894014] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_eventlog = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894173] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_journal = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894331] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_json = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894491] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_rootwrap_daemon = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894651] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_stderr = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894809] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] use_syslog = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.894966] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vcpu_pin_set = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895133] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plugging_is_fatal = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895300] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plugging_timeout = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895467] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] virt_mkfs = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895628] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] volume_usage_poll_interval = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895787] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] watch_log_file = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.895952] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] web = /usr/share/spice-html5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 479.896151] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_concurrency.disable_process_locking = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.896456] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.896643] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.896813] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.896985] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_metrics.metrics_process_name = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.897156] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.897323] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.897502] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.auth_strategy = keystone {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.897668] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.compute_link_prefix = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.897840] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898014] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.dhcp_domain = novalocal {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898181] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.enable_instance_password = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898353] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.glance_link_prefix = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898556] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898734] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.instance_list_cells_batch_strategy = distributed {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.898897] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.instance_list_per_project_cells = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899060] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.list_records_by_skipping_down_cells = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899228] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.local_metadata_per_cell = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899420] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.max_limit = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899598] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.metadata_cache_expiration = 15 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899775] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.neutron_default_tenant_id = default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.899945] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.response_validation = warn {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900126] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.use_neutron_default_nets = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900299] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900475] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_dynamic_failure_fatal = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900646] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900821] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_dynamic_ssl_certfile = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.900993] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_dynamic_targets = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.901156] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_jsonfile_path = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.901340] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api.vendordata_providers = ['StaticJSON'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.901537] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.backend = dogpile.cache.memcached {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.901707] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.backend_argument = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.901876] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.config_prefix = cache.oslo {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902046] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.dead_timeout = 60.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902209] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.debug_cache_backend = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902372] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.enable_retry_client = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902537] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.enable_socket_keepalive = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902707] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.enabled = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.902871] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.enforce_fips_mode = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903035] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.expiration_time = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903197] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.hashclient_retry_attempts = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903365] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.hashclient_retry_delay = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903519] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_dead_retry = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903679] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_password = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.903842] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904011] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904184] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_pool_maxsize = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904350] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_pool_unused_timeout = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904514] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_sasl_enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904693] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_servers = ['localhost:11211'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.904860] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_socket_timeout = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905019] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.memcache_username = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905184] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.proxies = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905351] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_db = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905513] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_password = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905680] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_sentinel_service_name = mymaster {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.905853] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906021] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_server = localhost:6379 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906186] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_socket_timeout = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906348] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.redis_username = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906514] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.retry_attempts = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906676] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.retry_delay = 0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.906840] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.socket_keepalive_count = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907004] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.socket_keepalive_idle = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907163] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.socket_keepalive_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907320] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.tls_allowed_ciphers = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907481] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.tls_cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907638] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.tls_certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907796] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.tls_enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.907953] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cache.tls_keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908142] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908320] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.auth_type = password {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908483] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908659] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.catalog_info = volumev3::publicURL {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908819] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.908982] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909145] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.cross_az_attach = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909318] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.debug = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909475] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.endpoint_template = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909642] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.http_retries = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909803] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.909960] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910128] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.os_region_name = RegionOne {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910377] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910530] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cinder.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910651] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910812] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.cpu_dedicated_set = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.910970] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.cpu_shared_set = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911133] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.image_type_exclude_list = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911300] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.live_migration_wait_for_vif_plug = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911459] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.max_concurrent_disk_ops = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911621] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.max_disk_devices_to_attach = -1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911780] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.911946] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912122] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.resource_provider_association_refresh = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912284] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912448] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.shutdown_retry_interval = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912623] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912799] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] conductor.workers = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.912971] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] console.allowed_origins = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913130] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] console.ssl_ciphers = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913300] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] console.ssl_minimum_version = default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913471] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] consoleauth.enforce_session_timeout = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913641] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] consoleauth.token_ttl = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913806] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.913964] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914124] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914285] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914444] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914599] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914758] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.914914] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915069] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915226] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915384] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915539] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915693] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.915858] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.service_type = accelerator {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916073] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916198] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916355] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916511] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916687] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.916847] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] cyborg.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917026] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.backend = sqlalchemy {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917194] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.connection = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917352] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.connection_debug = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917519] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.connection_parameters = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917684] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.connection_recycle_time = 3600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.917844] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.connection_trace = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918004] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.db_inc_retry_interval = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918166] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.db_max_retries = 20 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918326] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.db_max_retry_interval = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918488] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.db_retry_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918650] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.max_overflow = 50 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918809] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.max_pool_size = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.918966] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.max_retries = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919133] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.mysql_sql_mode = TRADITIONAL {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919318] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.mysql_wsrep_sync_wait = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919468] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.pool_timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919631] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.retry_interval = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.slave_connection = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.919948] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.sqlite_synchronous = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.920122] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] database.use_db_reconnect = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.920325] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.backend = sqlalchemy {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.920502] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.connection = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.920669] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.connection_debug = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.920835] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.connection_parameters = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921000] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.connection_recycle_time = 3600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921164] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.connection_trace = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921326] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.db_inc_retry_interval = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921492] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.db_max_retries = 20 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921653] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.db_max_retry_interval = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921815] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.db_retry_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.921974] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.max_overflow = 50 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922136] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.max_pool_size = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922297] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.max_retries = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922471] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922624] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.mysql_wsrep_sync_wait = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922782] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.pool_timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.922943] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.retry_interval = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923099] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.slave_connection = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923260] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] api_database.sqlite_synchronous = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923435] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] devices.enabled_mdev_types = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923611] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923782] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ephemeral_storage_encryption.default_format = luks {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.923945] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ephemeral_storage_encryption.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924121] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ephemeral_storage_encryption.key_size = 512 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924295] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.api_servers = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924461] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924625] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.924948] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925105] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925267] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.debug = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925440] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.default_trusted_certificate_ids = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925597] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.enable_certificate_validation = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925757] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.enable_rbd_download = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.925913] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926079] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926241] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926402] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926562] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926721] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.num_retries = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.926887] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.rbd_ceph_conf = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927051] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.rbd_connect_timeout = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927220] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.rbd_pool = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927383] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.rbd_user = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927543] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927699] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.927857] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928039] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.service_type = image {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928207] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928369] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928529] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928687] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.928867] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.929038] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.verify_glance_signatures = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.929196] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] glance.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.929394] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] guestfs.debug = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.929571] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] mks.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.929937] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.930131] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.manager_interval = 2400 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.930334] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.precache_concurrency = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.930538] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.remove_unused_base_images = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.930727] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.930902] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931081] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] image_cache.subdirectory_name = _base {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931258] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.api_max_retries = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931425] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.api_retry_interval = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931583] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931746] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.auth_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.931906] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932075] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932245] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932411] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.conductor_group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932574] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932735] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.932893] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933075] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933241] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933405] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933562] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933728] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.peer_list = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.933884] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934043] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934208] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.serial_console_state_timeout = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934368] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934540] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.service_type = baremetal {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934699] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.shard = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.934862] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935023] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935180] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935336] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935518] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935678] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ironic.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.935860] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936043] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] key_manager.fixed_key = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936232] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936397] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.barbican_api_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936557] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.barbican_endpoint = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936729] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.barbican_endpoint_type = public {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.936888] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.barbican_region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937045] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937203] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937366] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937527] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937682] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.937843] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.number_of_retries = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938001] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.retry_delay = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938160] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.send_service_user_token = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938321] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938477] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938634] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.verify_ssl = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938794] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican.verify_ssl_path = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.938960] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939120] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.auth_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939300] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939480] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939646] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939812] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.939968] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.940143] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.940337] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] barbican_service_user.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.940533] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.approle_role_id = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.940701] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.approle_secret_id = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.940877] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.kv_mountpoint = secret {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941041] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.kv_path = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941207] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.kv_version = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941366] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.namespace = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941525] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.root_token_id = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941683] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.ssl_ca_crt_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.941850] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.timeout = 60.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942011] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.use_ssl = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942179] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942349] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942512] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.auth_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942667] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942825] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.942986] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943143] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943299] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943458] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943617] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943777] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.943929] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944095] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944259] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944416] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944569] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944735] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.service_type = identity {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.944951] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945120] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945279] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945437] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945616] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945773] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] keystone.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.945973] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.connection_uri = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946132] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_mode = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946296] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_model_extra_flags = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946462] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_models = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946629] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_power_governor_high = performance {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946793] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_power_governor_low = powersave {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.946954] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_power_management = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.947153] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.947316] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.device_detach_attempts = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.947441] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.device_detach_timeout = 20 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.947606] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.disk_cachemodes = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.947763] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.disk_prefix = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948018] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.enabled_perf_events = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948191] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.file_backed_memory = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948356] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.gid_maps = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948515] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.hw_disk_discard = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948686] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.hw_machine_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.948919] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_rbd_ceph_conf = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949098] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949287] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949475] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_rbd_glance_store_name = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949645] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_rbd_pool = rbd {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949817] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_type = default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.949971] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.images_volume_group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950133] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.inject_key = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950293] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.inject_partition = -2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950486] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.inject_password = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950655] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.iscsi_iface = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950815] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.iser_use_multipath = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.950980] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_bandwidth = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951146] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_completion_timeout = 800 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951309] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_downtime = 500 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951473] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_downtime_delay = 75 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951635] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_downtime_steps = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951796] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_inbound_addr = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.951959] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_permit_auto_converge = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952133] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_permit_post_copy = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952301] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_scheme = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952477] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_timeout_action = abort {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952643] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_tunnelled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952803] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_uri = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.952966] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.live_migration_with_native_tls = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.953123] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.max_queues = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.953290] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.mem_stats_period_seconds = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.953528] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.953694] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.nfs_mount_options = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.954400] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.954589] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_aoe_discover_tries = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.954765] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_iser_scan_tries = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.954937] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_memory_encrypted_guests = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.955093] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_nvme_discover_tries = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.955263] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_pcie_ports = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.955436] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.num_volume_scan_tries = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.955609] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.pmem_namespaces = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.955780] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.quobyte_client_cfg = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956092] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956294] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rbd_connect_timeout = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956448] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956614] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956775] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rbd_secret_uuid = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.956939] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rbd_user = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957104] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.realtime_scheduler_priority = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957278] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.remote_filesystem_transport = ssh {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957439] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rescue_image_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957599] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rescue_kernel_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957757] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rescue_ramdisk_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.957924] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rng_dev_path = /dev/urandom {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.958081] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.rx_queue_size = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.958246] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.smbfs_mount_options = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.958527] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.958700] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.snapshot_compression = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.958864] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.snapshot_image_format = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959080] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959276] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.sparse_logical_volumes = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959446] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.swtpm_enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959619] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.swtpm_group = tss {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.swtpm_user = tss {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.959958] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.sysinfo_serial = unique {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.960136] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.tb_cache_size = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.960340] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.tx_queue_size = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.960519] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.uid_maps = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.960691] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.use_virtio_for_bridges = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.960864] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.virt_type = kvm {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961036] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.volume_clear = zero {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961204] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.volume_clear_size = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961373] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.volume_use_multipath = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961536] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_cache_path = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961705] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.961872] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_mount_group = qemu {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.962035] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_mount_opts = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.962200] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.962477] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.962656] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.vzstorage_mount_user = stack {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.962825] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963002] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963174] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.auth_type = password {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963334] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963497] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963668] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963842] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.963987] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964178] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.default_floating_pool = public {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964335] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964501] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.extension_sync_interval = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964667] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.http_retries = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964828] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.964995] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965150] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965319] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.metadata_proxy_shared_secret = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965481] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965650] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.ovs_bridge = br-int {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965815] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.physnets = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.965984] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.region_name = RegionOne {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966146] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966316] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.service_metadata_proxy = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966476] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966641] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.service_type = network {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966802] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.966961] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967120] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967279] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967460] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967621] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] neutron.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967795] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] notifications.bdms_in_notifications = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.967970] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] notifications.default_level = INFO {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968156] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] notifications.notification_format = unversioned {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968325] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] notifications.notify_on_state_change = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968501] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968672] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] pci.alias = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968838] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] pci.device_spec = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.968999] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] pci.report_in_placement = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.969170] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.969369] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.auth_type = password {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.969545] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.auth_url = http://10.180.1.21/identity {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.969706] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.969866] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970029] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970190] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970399] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970575] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.default_domain_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970736] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.default_domain_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.970893] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.domain_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971051] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.domain_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971209] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971371] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971533] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971689] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.971846] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972018] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.password = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972182] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.project_domain_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972347] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.project_domain_name = Default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972513] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.project_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972690] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.project_name = service {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.972934] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.region_name = RegionOne {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.973114] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.973280] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.973451] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.service_type = placement {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.973653] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.973858] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.974028] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.974188] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.system_scope = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.974349] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.974510] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.trust_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.974668] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.user_domain_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975137] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.user_domain_name = Default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975137] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.user_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975324] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.username = nova {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975507] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975667] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] placement.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.975850] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.cores = 20 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976024] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.count_usage_from_placement = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976200] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976373] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.injected_file_content_bytes = 10240 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976541] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.injected_file_path_length = 255 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976703] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.injected_files = 5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.976869] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.instances = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977033] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.key_pairs = 100 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977200] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.metadata_items = 128 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977367] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.ram = 51200 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977530] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.recheck_quota = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977696] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.server_group_members = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.977859] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] quota.server_groups = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978033] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978196] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978358] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.image_metadata_prefilter = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978522] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978685] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.max_attempts = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.978849] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.max_placement_results = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979013] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979178] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.query_placement_for_image_type_support = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979361] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979543] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] scheduler.workers = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979717] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.979887] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.980119] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.980258] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.980438] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.980613] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.980776] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981097] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981284] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.host_subset_size = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981454] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981616] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.image_properties_default_architecture = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981780] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.981948] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.isolated_hosts = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982112] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.isolated_images = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982276] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.max_instances_per_host = 50 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982447] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982610] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982772] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.pci_in_placement = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.982937] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983100] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983262] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983424] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983585] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983746] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.983906] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.track_instance_changes = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.984094] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.984269] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metrics.required = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.984436] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metrics.weight_multiplier = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.984798] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metrics.weight_of_unavailable = -10000.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.984798] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] metrics.weight_setting = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985070] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985249] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985426] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.port_range = 10000:20000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985598] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985766] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.985944] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] serial_console.serialproxy_port = 6083 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986116] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986288] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.auth_type = password {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986452] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986607] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986768] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.986927] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987081] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987250] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.send_service_user_token = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987411] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987566] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] service_user.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987744] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.agent_enabled = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.987896] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.988224] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.988439] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.html5proxy_host = 0.0.0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.988610] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.html5proxy_port = 6082 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.988774] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.image_compression = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.988933] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.jpeg_compression = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.989092] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.playback_compression = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.989281] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.server_listen = 127.0.0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.989476] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.989638] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.streaming_mode = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.989793] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] spice.zlib_compression = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990134] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] upgrade_levels.baseapi = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990134] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] upgrade_levels.compute = auto {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990271] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] upgrade_levels.conductor = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990454] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] upgrade_levels.scheduler = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990630] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990796] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.auth_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.990952] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991113] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991278] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991439] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991621] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991800] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.991960] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vendordata_dynamic_auth.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992153] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.api_retry_count = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992317] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.ca_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992489] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.cache_prefix = devstack-image-cache {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992655] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.cluster_name = testcl1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992819] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.connection_pool_size = 10 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.992977] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.console_delay_seconds = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.993143] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.datastore_regex = ^datastore.* {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.993355] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.993528] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.host_password = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.993696] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.host_port = 443 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.993862] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.host_username = administrator@vsphere.local {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994028] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.insecure = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994187] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.integration_bridge = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994353] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.maximum_objects = 100 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994512] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.pbm_default_policy = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994671] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.pbm_enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994829] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.pbm_wsdl_location = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.994995] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995153] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.serial_port_proxy_uri = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995310] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.serial_port_service_uri = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995477] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.task_poll_interval = 0.5 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995646] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.use_linked_clone = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995813] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.vnc_keymap = en-us {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.995975] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.vnc_port = 5900 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.996155] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vmware.vnc_port_total = 10000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.996343] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.auth_schemes = ['none'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.996519] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.996815] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.996996] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.997166] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.novncproxy_port = 6080 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.997367] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.server_listen = 127.0.0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.997546] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.997706] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.vencrypt_ca_certs = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.997864] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.vencrypt_client_cert = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998020] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vnc.vencrypt_client_key = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998194] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998362] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_deep_image_inspection = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998526] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_fallback_pcpu_query = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998687] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_group_policy_check_upcall = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.998851] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999013] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.disable_rootwrap = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999176] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.enable_numa_live_migration = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999368] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999542] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999706] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.handle_virt_lifecycle_events = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 479.999867] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.libvirt_disable_apic = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000038] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.never_download_image_if_on_rbd = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000208] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000389] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000566] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000727] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.000886] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001047] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001209] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001371] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001540] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001747] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.001921] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.client_socket_timeout = 900 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002088] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.default_pool_size = 1000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002253] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.keep_alive = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002421] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.max_header_line = 16384 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002583] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.secure_proxy_ssl_header = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002744] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.ssl_ca_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.002904] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.ssl_cert_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.003062] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.ssl_key_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.003226] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.tcp_keepidle = 600 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.003398] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.003561] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] zvm.ca_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.003720] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] zvm.cloud_connector_url = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004067] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004257] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] zvm.reachable_timeout = 300 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004444] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.enforce_new_defaults = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004614] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.enforce_scope = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.policy_default_rule = default {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.004968] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005140] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.policy_file = policy.yaml {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005308] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005467] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005624] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005783] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.005944] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006110] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006283] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006457] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.connection_string = messaging:// {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006621] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.enabled = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006788] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.es_doc_type = notification {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.006948] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.es_scroll_size = 10000 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007112] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.es_scroll_time = 2m {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007293] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.filter_error_trace = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007474] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.hmac_keys = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007642] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.sentinel_service_name = mymaster {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007809] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.socket_timeout = 0.1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.007969] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.trace_requests = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.008143] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler.trace_sqlalchemy = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.008321] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler_jaeger.process_tags = {} {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.008480] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler_jaeger.service_name_prefix = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.008641] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] profiler_otlp.service_name_prefix = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.008804] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] remote_debug.host = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009040] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] remote_debug.port = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009264] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009422] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009594] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009756] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.009916] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010074] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010231] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010414] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010586] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010756] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.hostname = devstack {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.010986] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.011167] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.011337] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.011507] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.011707] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.011878] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012258] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012258] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012371] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012532] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012697] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.012858] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013018] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013183] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013345] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013506] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013666] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013824] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.013987] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014152] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014332] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014500] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014659] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014825] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.014991] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.ssl_version = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.015152] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.015337] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.015503] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_notifications.retry = -1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.015683] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.015855] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_messaging_notifications.transport_url = **** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016039] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.auth_section = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016203] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.auth_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016360] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.cafile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016516] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.certfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016673] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.collect_timing = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016828] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.connect_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.016981] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.connect_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017134] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.endpoint_id = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017311] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.endpoint_override = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017485] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.insecure = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017639] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.keyfile = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017791] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.max_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.017942] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.min_version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018100] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.region_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018258] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.retriable_status_codes = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018416] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.service_name = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018571] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.service_type = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018731] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.split_loggers = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.018888] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.status_code_retries = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019044] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.status_code_retry_delay = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019200] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.timeout = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019389] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.valid_interfaces = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019553] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_limit.version = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019719] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_reports.file_event_handler = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.019882] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_reports.file_event_handler_interval = 1 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020077] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] oslo_reports.log_dir = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020262] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020424] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020581] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020742] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.020907] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021062] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_linux_bridge_privileged.user = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021230] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021388] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021549] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.helper_command = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021727] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.021892] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022049] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] vif_plug_ovs_privileged.user = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022217] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.flat_interface = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022392] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022561] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022727] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.022893] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023055] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023217] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023375] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_linux_bridge.vlan_interface = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023549] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023715] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.isolate_vif = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.023887] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024063] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024234] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024402] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.ovsdb_interface = native {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024560] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_vif_ovs.per_port_bridge = False {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024722] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] os_brick.lock_path = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.024889] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.capabilities = [21] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025043] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025194] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.helper_command = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025356] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025515] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.thread_pool_size = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025669] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] privsep_osbrick.user = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.025953] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026144] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.group = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026306] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.helper_command = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026475] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026637] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.thread_pool_size = 8 {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026795] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] nova_sys_admin.user = None {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 480.026926] env[62736]: DEBUG oslo_service.service [None req-e8cedcc0-cbb6-408d-9c45-f93ae64b3ffd None None] ******************************************************************************** {{(pid=62736) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 480.027360] env[62736]: INFO nova.service [-] Starting compute node (version 0.0.1) [ 480.028334] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg f39e7872e73a4aeb9e7caef6e96457b5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 480.036514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f39e7872e73a4aeb9e7caef6e96457b5 [ 480.531315] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Getting list of instances from cluster (obj){ [ 480.531315] env[62736]: value = "domain-c8" [ 480.531315] env[62736]: _type = "ClusterComputeResource" [ 480.531315] env[62736]: } {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 480.532389] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fed0841-2ac6-4b00-a860-84f07314fbc8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 480.541340] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Got total of 0 instances {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 480.541921] env[62736]: WARNING nova.virt.vmwareapi.driver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 480.542413] env[62736]: INFO nova.virt.node [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Generated node identity 0c9afe22-9d34-458c-8118-58661faecbae [ 480.542661] env[62736]: INFO nova.virt.node [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Wrote node identity 0c9afe22-9d34-458c-8118-58661faecbae to /opt/stack/data/n-cpu-1/compute_id [ 480.543084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg be5402116bba4f1389bcce2e12c9ea5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 480.556036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be5402116bba4f1389bcce2e12c9ea5c [ 481.047373] env[62736]: WARNING nova.compute.manager [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Compute nodes ['0c9afe22-9d34-458c-8118-58661faecbae'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 481.048138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 4ef5cb2ee84244bfb1eff3f5d6439c82 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 481.074121] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ef5cb2ee84244bfb1eff3f5d6439c82 [ 481.551185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 4c62c10e410a4d129128b363a0cde91e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 481.562880] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c62c10e410a4d129128b363a0cde91e [ 482.053841] env[62736]: INFO nova.compute.manager [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 482.054452] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 2ddb12c5becd4e1fa301f3ccfe950673 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 482.067669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ddb12c5becd4e1fa301f3ccfe950673 [ 482.557875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 59778ffcb59642dc924423d60a874ca2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 482.569765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59778ffcb59642dc924423d60a874ca2 [ 483.060030] env[62736]: WARNING nova.compute.manager [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 483.060400] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 483.060693] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 483.060918] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 483.061141] env[62736]: DEBUG nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 483.062111] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f47eb0e2-4593-4492-b7c4-b683ec28d11c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 483.070023] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1ed0d4c-094d-4995-9bc3-a439b82b1e6a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 483.084460] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-735cc8f9-13f8-48c3-8561-a5660cf28801 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 483.090695] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f5352f4-5ad8-41f0-bba3-6830e817b39d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 483.119102] env[62736]: DEBUG nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181753MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 483.119149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 483.119396] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 483.119711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 632799afb03b4c0b8021a2bd68d85469 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 483.131296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 632799afb03b4c0b8021a2bd68d85469 [ 483.621648] env[62736]: WARNING nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] No compute node record for cpu-1:0c9afe22-9d34-458c-8118-58661faecbae: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host 0c9afe22-9d34-458c-8118-58661faecbae could not be found. [ 483.622942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 1faedf93ce5e4624afcebbf33a10740e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 483.634543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1faedf93ce5e4624afcebbf33a10740e [ 484.125312] env[62736]: INFO nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: 0c9afe22-9d34-458c-8118-58661faecbae [ 484.125818] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg b7be324353174b54806cbb2827b4a1fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 484.136067] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b7be324353174b54806cbb2827b4a1fa [ 484.628676] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg f8d8e08256a646a8a34f8d0e43ec0a1a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 484.648390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8d8e08256a646a8a34f8d0e43ec0a1a [ 485.131833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 1de2bdb9545f40f8a3f651793df54672 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 485.153441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1de2bdb9545f40f8a3f651793df54672 [ 485.634825] env[62736]: DEBUG nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 485.635164] env[62736]: DEBUG nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 485.794094] env[62736]: INFO nova.scheduler.client.report [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] [req-09e857fc-2eaf-48e9-adbd-fa029db12187] Created resource provider record via placement API for resource provider with UUID 0c9afe22-9d34-458c-8118-58661faecbae and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 485.810749] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7280b6f3-ea19-4c05-bdcd-93535f738302 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 485.818601] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8b26069-60da-4f01-8066-cb6c12454db9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 485.848418] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08ab1497-6276-49b6-8a34-47cf27630aab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 485.855724] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1218e275-934a-4e0d-9a45-8eb22242fb4f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 485.868405] env[62736]: DEBUG nova.compute.provider_tree [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Updating inventory in ProviderTree for provider 0c9afe22-9d34-458c-8118-58661faecbae with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 485.868973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg 7d17318cc1b34fc396804ce9cbe03dd3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 485.876506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d17318cc1b34fc396804ce9cbe03dd3 [ 486.406566] env[62736]: DEBUG nova.scheduler.client.report [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Updated inventory for provider 0c9afe22-9d34-458c-8118-58661faecbae with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:971}} [ 486.406810] env[62736]: DEBUG nova.compute.provider_tree [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Updating resource provider 0c9afe22-9d34-458c-8118-58661faecbae generation from 0 to 1 during operation: update_inventory {{(pid=62736) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 486.406949] env[62736]: DEBUG nova.compute.provider_tree [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Updating inventory in ProviderTree for provider 0c9afe22-9d34-458c-8118-58661faecbae with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 486.455127] env[62736]: DEBUG nova.compute.provider_tree [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Updating resource provider 0c9afe22-9d34-458c-8118-58661faecbae generation from 1 to 2 during operation: update_traits {{(pid=62736) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 486.457463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Expecting reply to msg e502e8fb2a684350897f3e61922a3949 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 486.481207] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e502e8fb2a684350897f3e61922a3949 [ 486.959816] env[62736]: DEBUG nova.compute.resource_tracker [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 486.960252] env[62736]: DEBUG oslo_concurrency.lockutils [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.841s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 486.960252] env[62736]: DEBUG nova.service [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Creating RPC server for service compute {{(pid=62736) start /opt/stack/nova/nova/service.py:186}} [ 486.971032] env[62736]: INFO oslo.messaging._drivers.impl_rabbit [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Creating fanout queue: compute_fanout_34d4159565a24706903fa338f96f1524 [ 486.973964] env[62736]: DEBUG nova.service [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] Join ServiceGroup membership for this service compute {{(pid=62736) start /opt/stack/nova/nova/service.py:203}} [ 486.974133] env[62736]: DEBUG nova.servicegroup.drivers.db [None req-02fe794d-dd9f-4866-b75e-384013a52725 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=62736) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 491.977108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2800c42a0ad84801ad6b0bba5a68acf2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 491.988998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2800c42a0ad84801ad6b0bba5a68acf2 [ 516.975870] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._sync_power_states {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 516.976508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 89136f69808b4568b85556903995fdfc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 516.988747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89136f69808b4568b85556903995fdfc [ 517.480467] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Getting list of instances from cluster (obj){ [ 517.480467] env[62736]: value = "domain-c8" [ 517.480467] env[62736]: _type = "ClusterComputeResource" [ 517.480467] env[62736]: } {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 517.481585] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95e48002-267d-4934-8364-edcc51d0bdd9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 517.490069] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Got total of 0 instances {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 517.490296] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 517.490613] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Getting list of instances from cluster (obj){ [ 517.490613] env[62736]: value = "domain-c8" [ 517.490613] env[62736]: _type = "ClusterComputeResource" [ 517.490613] env[62736]: } {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 517.491476] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e415a8e-83cb-4ef0-bcef-88a033ecc5a1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 517.498665] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Got total of 0 instances {{(pid=62736) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 524.662504] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "01127e18-3d68-4b50-b0d4-89f186aa7fb4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 524.662829] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "01127e18-3d68-4b50-b0d4-89f186aa7fb4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 524.663199] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg edb784bf23384d05b7b9cc994e539335 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 524.688181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edb784bf23384d05b7b9cc994e539335 [ 525.166260] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 525.167956] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d9b646d27c314668b2bb9347654878ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 525.276000] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9b646d27c314668b2bb9347654878ce [ 525.500614] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "d995b3ce-c5b5-490e-b774-706a74632f27" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.500614] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "d995b3ce-c5b5-490e-b774-706a74632f27" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.500614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 6a7a799b28eb42cc8b433594eb5f1e59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 525.539585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a7a799b28eb42cc8b433594eb5f1e59 [ 525.710478] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.710772] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.712571] env[62736]: INFO nova.compute.claims [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 525.714338] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d57a3b3060454e24b064f3ba74eccaec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 525.790657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d57a3b3060454e24b064f3ba74eccaec [ 526.008287] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 526.008287] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg e8a2e2015b1a42ac81e35a5a155d49a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.046909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8a2e2015b1a42ac81e35a5a155d49a6 [ 526.224260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 30a845d1f32e49789079d31831ba1033 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.245188] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30a845d1f32e49789079d31831ba1033 [ 526.377521] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquiring lock "03b95ad5-85fe-41ad-987f-8140380f0a0d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 526.377521] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "03b95ad5-85fe-41ad-987f-8140380f0a0d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 526.377521] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 996c3a7a155b44a98099c8137111ba9d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.387195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 996c3a7a155b44a98099c8137111ba9d [ 526.523183] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 526.708229] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquiring lock "1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 526.708770] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 526.709540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 2f55b342fd3f4f76871be6fbe28b7b11 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.727272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f55b342fd3f4f76871be6fbe28b7b11 [ 526.857842] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef41b062-5832-429a-9fba-f102a9bd0f20 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 526.867389] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49453e70-056b-4d73-995f-20f08f05e2b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 526.903311] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 526.905174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 2088a2a1ddb74ac4bae91b2b6b9aaf25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.907137] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b14147c-785d-4d8f-b24b-091cf089c8aa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 526.913541] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "2eb46a2a-27fc-4d44-9d18-6d4553026a60" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 526.914042] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "2eb46a2a-27fc-4d44-9d18-6d4553026a60" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 526.914565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg ab9aca5c659b467d9f95bc1dc7ea1c3b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.924690] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8a71bfa-b885-4f78-8805-79e6e1f890e4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 526.928152] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab9aca5c659b467d9f95bc1dc7ea1c3b [ 526.949473] env[62736]: DEBUG nova.compute.provider_tree [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 526.950237] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg c2a99b40ac524c6a8faa67dd5df70d8d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 526.969558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2088a2a1ddb74ac4bae91b2b6b9aaf25 [ 526.974085] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2a99b40ac524c6a8faa67dd5df70d8d [ 527.213165] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 527.215156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg a556d8102a49427f945498038404b281 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 527.258485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a556d8102a49427f945498038404b281 [ 527.423107] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 527.424470] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg dab7acb67f23493d96a8d27e747cd486 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 527.426188] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.452975] env[62736]: DEBUG nova.scheduler.client.report [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 527.455445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 607e3d333cc04dd2bc8412634a9a9777 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 527.467971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dab7acb67f23493d96a8d27e747cd486 [ 527.472706] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 607e3d333cc04dd2bc8412634a9a9777 [ 527.714476] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.714708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.715161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 81814fe5d6f54e6787e762429a9a883e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 527.725658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81814fe5d6f54e6787e762429a9a883e [ 527.738247] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.941392] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.964174] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.248s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.964174] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 527.964174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 69e3a19adf9f457aa36e66d58611d4c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 527.964174] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.439s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.964174] env[62736]: INFO nova.compute.claims [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 527.966024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 7b33590738314e9abf04d3ae6e118b4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 528.014043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69e3a19adf9f457aa36e66d58611d4c1 [ 528.015007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b33590738314e9abf04d3ae6e118b4c [ 528.217074] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 528.218779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 30a660b808344a10bb80f5aece1f3910 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 528.263692] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30a660b808344a10bb80f5aece1f3910 [ 528.469617] env[62736]: DEBUG nova.compute.utils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 528.470352] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 0996c549e74b4501858b59ee280898e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 528.471270] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 528.471484] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 528.477226] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 2700c45ebae44d49bdc455ede420ffc1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 528.489982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2700c45ebae44d49bdc455ede420ffc1 [ 528.490552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0996c549e74b4501858b59ee280898e9 [ 528.739161] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.873345] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquiring lock "0b211be4-1955-456d-bf30-5195b5169807" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.873567] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "0b211be4-1955-456d-bf30-5195b5169807" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.874007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg ecad5675ce5c44caa74cd8e2213738a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 528.904681] env[62736]: DEBUG nova.policy [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c43c33165931465494c0a7da171126e4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de11ac365f704c2986c782a5918f802f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 528.909587] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ecad5675ce5c44caa74cd8e2213738a0 [ 528.976747] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 528.978684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg a8e4d5bac2024d1687b5138a9d2fe569 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.036723] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8e4d5bac2024d1687b5138a9d2fe569 [ 529.087166] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6184a48b-c687-4b6c-9c0b-78dea840f8e3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 529.094784] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94b8d919-94c5-475e-a14e-4acb85444c88 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 529.127575] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ccc7d5b-f2b7-45f1-b95f-d5c01f3d382c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 529.141869] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2ccd365-b206-44a2-9aac-7c70f9bbd883 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 529.157462] env[62736]: DEBUG nova.compute.provider_tree [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 529.157462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 16000399a24f4b5483821ba39adc6aa3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.164378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16000399a24f4b5483821ba39adc6aa3 [ 529.377525] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 529.377998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg fca07cd1a73b461385d94fc9bfd3483c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.455036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fca07cd1a73b461385d94fc9bfd3483c [ 529.485713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 33577f42e9554c52a922af26cab147fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.528426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33577f42e9554c52a922af26cab147fe [ 529.605085] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "ae1041c6-888a-4cb3-8da3-00e43b5b6531" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.605374] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "ae1041c6-888a-4cb3-8da3-00e43b5b6531" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.609114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 7b776fe8fa82481b879a68a18b19a4fb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.631113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b776fe8fa82481b879a68a18b19a4fb [ 529.650437] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Successfully created port: a989424f-fd97-4857-9982-210001ac2adc {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 529.660198] env[62736]: DEBUG nova.scheduler.client.report [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 529.662525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 2a7743e01de340298aac5556ffbe968e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 529.678384] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a7743e01de340298aac5556ffbe968e [ 529.905799] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.988915] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 530.016404] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 530.016653] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 530.016806] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 530.017007] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 530.017230] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 530.017323] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 530.017559] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 530.017718] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 530.019139] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 530.019344] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 530.019529] env[62736]: DEBUG nova.virt.hardware [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 530.020443] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a509f5b-d6f7-48f3-a093-0091646f4c4c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 530.028794] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-410febb7-17f3-4c9a-a4a5-e06a122202ba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 530.049526] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0a1d958-e5a7-4367-983b-f10a1ae322e9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 530.112174] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 530.113846] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 611af5ecee1c4562b5418afa82102c21 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.157944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 611af5ecee1c4562b5418afa82102c21 [ 530.165581] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.204s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 530.166206] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 530.167946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 81a0defd092c4ba59788be2ec98fe874 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.168954] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.743s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 530.170359] env[62736]: INFO nova.compute.claims [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 530.171904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 5d91e1aac8c341fc8d52cbf3516aa887 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.198365] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81a0defd092c4ba59788be2ec98fe874 [ 530.216858] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d91e1aac8c341fc8d52cbf3516aa887 [ 530.544809] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "aa9671f3-07a8-422f-93b5-006de0f2cb3c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 530.545026] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "aa9671f3-07a8-422f-93b5-006de0f2cb3c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 530.545561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 41ad62ec7b004db9a844e64a7d0365c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.557467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41ad62ec7b004db9a844e64a7d0365c3 [ 530.637089] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 530.676595] env[62736]: DEBUG nova.compute.utils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 530.676811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 62b27eff628f4f768170f16591417206 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.680146] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 530.680146] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 530.682493] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 6991c79fd83a454db3a96994d1f8b4e4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 530.696278] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6991c79fd83a454db3a96994d1f8b4e4 [ 530.755599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62b27eff628f4f768170f16591417206 [ 530.884124] env[62736]: DEBUG nova.policy [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a9a0e0bbae8046a783b171c8a4ee3c11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f9f409d646e3429cac4cd3555a5073c2', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 531.048764] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 531.050546] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 8a60df570522468da5b970f8d401e8bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.092938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a60df570522468da5b970f8d401e8bf [ 531.186953] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 531.188541] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 37f12919bdca4aa9b35b826545038826 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.225154] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37f12919bdca4aa9b35b826545038826 [ 531.344830] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae329ea2-ec72-4467-ac9f-cc97f6821851 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 531.353058] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36c88c0e-1366-4c20-b0e5-42aa3cf250db {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 531.387082] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-383bb489-4671-43b6-94fb-793df4573c7d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 531.396703] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e91c934-a1c7-4fea-aac3-62710cbb2de2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 531.410641] env[62736]: DEBUG nova.compute.provider_tree [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 531.412720] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 64814b318ba5494e8ec5b170c01519cd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.425928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64814b318ba5494e8ec5b170c01519cd [ 531.426963] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Successfully created port: fdbabc59-acd2-490d-95ce-d13232dc4eb2 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 531.432767] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "a5a690d3-8018-4cc5-8d59-55062a595ed0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.433109] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "a5a690d3-8018-4cc5-8d59-55062a595ed0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.433560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 165e4584af284be5b00b9686dc49ca7a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.449746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 165e4584af284be5b00b9686dc49ca7a [ 531.569458] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.693026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 4e67f64318f1436b87ab057e074b4282 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.735525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e67f64318f1436b87ab057e074b4282 [ 531.915246] env[62736]: DEBUG nova.scheduler.client.report [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 531.917697] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg b93ae450ebd64988851925526707f0ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.933005] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b93ae450ebd64988851925526707f0ca [ 531.934912] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 531.940209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 3628f359a2cd449e9f573d03d0c4cc53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 531.969990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3628f359a2cd449e9f573d03d0c4cc53 [ 532.196518] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 532.222925] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 532.223423] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 532.223423] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 532.223543] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 532.223625] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 532.223756] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 532.223950] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 532.224143] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 532.224440] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 532.224565] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 532.224644] env[62736]: DEBUG nova.virt.hardware [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 532.225503] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-997f7ce7-bdd7-4f5e-b3ed-b34a6e9316e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 532.233715] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3b97939-6a08-4557-9363-48150e089e61 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 532.285240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "01b348db-6a0d-4283-b9d9-8edacfd1c219" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.285483] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "01b348db-6a0d-4283-b9d9-8edacfd1c219" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.419845] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.251s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.420412] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 532.423994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg c9eaec98439d4dbd80448e333f92ddb6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 532.423994] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 4.685s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.424430] env[62736]: INFO nova.compute.claims [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 532.425899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 8c1e80cf8fe84e98bebe91ad9e8072bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 532.463565] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.472172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9eaec98439d4dbd80448e333f92ddb6 [ 532.477774] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c1e80cf8fe84e98bebe91ad9e8072bd [ 532.929400] env[62736]: DEBUG nova.compute.utils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 532.930092] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 51d21aaa40714cd58a2a62042a8b53ba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 532.932118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 4c2c4b3db3be45cabd87436381a25d1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 532.934559] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 532.934762] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 532.941244] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c2c4b3db3be45cabd87436381a25d1e [ 532.946095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51d21aaa40714cd58a2a62042a8b53ba [ 533.126525] env[62736]: DEBUG nova.policy [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0e2b5df479b642c68c88c8608359ea4d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f9b8a0b9692647368b3e08b6219291de', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 533.178124] env[62736]: ERROR nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 533.178124] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.178124] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.178124] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.178124] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.178124] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.178124] env[62736]: ERROR nova.compute.manager raise self.value [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.178124] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 533.178124] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.178124] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 533.178650] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.178650] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 533.178650] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 533.178650] env[62736]: ERROR nova.compute.manager [ 533.178650] env[62736]: Traceback (most recent call last): [ 533.178650] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 533.178650] env[62736]: listener.cb(fileno) [ 533.178650] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 533.178650] env[62736]: result = function(*args, **kwargs) [ 533.178650] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 533.178650] env[62736]: return func(*args, **kwargs) [ 533.178650] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 533.178650] env[62736]: raise e [ 533.178650] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.178650] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 533.178650] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.178650] env[62736]: created_port_ids = self._update_ports_for_instance( [ 533.178650] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.178650] env[62736]: with excutils.save_and_reraise_exception(): [ 533.178650] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.178650] env[62736]: self.force_reraise() [ 533.178650] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.178650] env[62736]: raise self.value [ 533.178650] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.178650] env[62736]: updated_port = self._update_port( [ 533.178650] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.178650] env[62736]: _ensure_no_port_binding_failure(port) [ 533.178650] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.178650] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 533.179558] env[62736]: nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 533.179558] env[62736]: Removing descriptor: 16 [ 533.179952] env[62736]: ERROR nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Traceback (most recent call last): [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] yield resources [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.driver.spawn(context, instance, image_meta, [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self._vmops.spawn(context, instance, image_meta, injected_files, [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] vm_ref = self.build_virtual_machine(instance, [ 533.179952] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] vif_infos = vmwarevif.get_vif_info(self._session, [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] for vif in network_info: [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self._sync_wrapper(fn, *args, **kwargs) [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.wait() [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self[:] = self._gt.wait() [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self._exit_event.wait() [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 533.180285] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] result = hub.switch() [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self.greenlet.switch() [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] result = function(*args, **kwargs) [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return func(*args, **kwargs) [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise e [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] nwinfo = self.network_api.allocate_for_instance( [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] created_port_ids = self._update_ports_for_instance( [ 533.180668] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] with excutils.save_and_reraise_exception(): [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.force_reraise() [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise self.value [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] updated_port = self._update_port( [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] _ensure_no_port_binding_failure(port) [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise exception.PortBindingFailed(port_id=port['id']) [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 533.181039] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] [ 533.181427] env[62736]: INFO nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Terminating instance [ 533.182639] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 533.182806] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 533.182967] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 533.183383] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 19eae0d7dcc34fed848a26857771b07f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.192469] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19eae0d7dcc34fed848a26857771b07f [ 533.440953] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 533.440953] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 93d954cd572a4a1b8126106ca36d628b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.479852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93d954cd572a4a1b8126106ca36d628b [ 533.551935] env[62736]: ERROR nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 533.551935] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.551935] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.551935] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.551935] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.551935] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.551935] env[62736]: ERROR nova.compute.manager raise self.value [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.551935] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 533.551935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.551935] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 533.552838] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.552838] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 533.552838] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 533.552838] env[62736]: ERROR nova.compute.manager [ 533.552838] env[62736]: Traceback (most recent call last): [ 533.552838] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 533.552838] env[62736]: listener.cb(fileno) [ 533.552838] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 533.552838] env[62736]: result = function(*args, **kwargs) [ 533.552838] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 533.552838] env[62736]: return func(*args, **kwargs) [ 533.552838] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 533.552838] env[62736]: raise e [ 533.552838] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.552838] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 533.552838] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.552838] env[62736]: created_port_ids = self._update_ports_for_instance( [ 533.552838] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.552838] env[62736]: with excutils.save_and_reraise_exception(): [ 533.552838] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.552838] env[62736]: self.force_reraise() [ 533.552838] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.552838] env[62736]: raise self.value [ 533.552838] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.552838] env[62736]: updated_port = self._update_port( [ 533.552838] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.552838] env[62736]: _ensure_no_port_binding_failure(port) [ 533.552838] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.552838] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 533.553880] env[62736]: nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 533.553880] env[62736]: Removing descriptor: 15 [ 533.553880] env[62736]: ERROR nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Traceback (most recent call last): [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] yield resources [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.driver.spawn(context, instance, image_meta, [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 533.553880] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] vm_ref = self.build_virtual_machine(instance, [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] vif_infos = vmwarevif.get_vif_info(self._session, [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] for vif in network_info: [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self._sync_wrapper(fn, *args, **kwargs) [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.wait() [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self[:] = self._gt.wait() [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self._exit_event.wait() [ 533.554329] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] result = hub.switch() [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self.greenlet.switch() [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] result = function(*args, **kwargs) [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return func(*args, **kwargs) [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise e [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] nwinfo = self.network_api.allocate_for_instance( [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 533.554693] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] created_port_ids = self._update_ports_for_instance( [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] with excutils.save_and_reraise_exception(): [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.force_reraise() [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise self.value [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] updated_port = self._update_port( [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] _ensure_no_port_binding_failure(port) [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 533.555048] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise exception.PortBindingFailed(port_id=port['id']) [ 533.555389] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 533.555389] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] [ 533.555389] env[62736]: INFO nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Terminating instance [ 533.555389] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 533.555389] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 533.555389] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 533.555389] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 3c33c3bc357446b784d809a348825da9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.558849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c33c3bc357446b784d809a348825da9 [ 533.681511] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6d30047-8f90-47ab-9914-628abc4411b0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 533.690781] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db09c40d-46d8-478e-90d8-975191f168a3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 533.721402] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0dc4e9a9-9103-46a5-b0e8-ca7cd01b532b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 533.728638] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4046001d-0c6f-4637-889f-29e21443676e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 533.744281] env[62736]: DEBUG nova.compute.provider_tree [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 533.745383] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 213352b7457c4afab631d4e17283294a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.754780] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 213352b7457c4afab631d4e17283294a [ 533.766516] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 533.953152] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 6d1f907fe1f040fb9cf0aa85386f581a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.972415] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 533.972950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg a8d3544510f14c5c9eb1a3f73cf2fef7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 533.991110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8d3544510f14c5c9eb1a3f73cf2fef7 [ 534.014731] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d1f907fe1f040fb9cf0aa85386f581a [ 534.091692] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 534.101174] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Successfully created port: b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 534.247767] env[62736]: DEBUG nova.scheduler.client.report [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 534.250242] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 51029da13779493ea2fe86890b7a05da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.262047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51029da13779493ea2fe86890b7a05da [ 534.370399] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 534.370933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 0753cdddf7874681add8001d66304af0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.389856] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0753cdddf7874681add8001d66304af0 [ 534.449164] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquiring lock "958e85c0-cb75-4dfd-92c4-b68957e87ec5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 534.450008] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "958e85c0-cb75-4dfd-92c4-b68957e87ec5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 534.458242] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 534.476029] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 534.476029] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 534.476226] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 534.476376] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b479dbe4-4965-459a-9493-a1726247ee00 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.485059] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07987345-f371-4a15-b84c-df7186f6fd49 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.501924] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 534.502137] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 534.502698] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 534.502698] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 534.502698] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 534.503185] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 534.503185] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 534.503185] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 534.503282] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 534.504077] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 534.504077] env[62736]: DEBUG nova.virt.hardware [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 534.504354] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4840c1f2-b66d-43b7-9716-fa8dc4976e6e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.522242] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d995b3ce-c5b5-490e-b774-706a74632f27 could not be found. [ 534.522242] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 534.522242] env[62736]: INFO nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Took 0.05 seconds to destroy the instance on the hypervisor. [ 534.522242] env[62736]: DEBUG oslo.service.loopingcall [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 534.522875] env[62736]: DEBUG nova.compute.manager [-] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 534.522979] env[62736]: DEBUG nova.network.neutron [-] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 534.532326] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03f2afa2-d089-49e7-8015-7bc587ddbe37 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.549966] env[62736]: DEBUG nova.network.neutron [-] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 534.550895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 04cc0b47ad3040c2ad1cbd169ddc94a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.561447] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04cc0b47ad3040c2ad1cbd169ddc94a9 [ 534.754307] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.330s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 534.754307] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 534.755567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 1cd71ba2135448499564fd4480425440 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.756567] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 6.815s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 534.757945] env[62736]: INFO nova.compute.claims [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 534.759430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 9bd1f8d15bf646cda68a47e373d85338 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.796238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cd71ba2135448499564fd4480425440 [ 534.802523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bd1f8d15bf646cda68a47e373d85338 [ 534.876588] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 534.876588] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 534.876588] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 534.876588] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-66a01483-97ba-4257-a40f-00aa5345ca8e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.882679] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-463aed19-a2d8-4aed-a3a1-8f001aced32a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.908442] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 01127e18-3d68-4b50-b0d4-89f186aa7fb4 could not be found. [ 534.908668] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 534.908840] env[62736]: INFO nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 534.909080] env[62736]: DEBUG oslo.service.loopingcall [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 534.909309] env[62736]: DEBUG nova.compute.manager [-] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 534.909401] env[62736]: DEBUG nova.network.neutron [-] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 534.944374] env[62736]: DEBUG nova.compute.manager [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Received event network-changed-fdbabc59-acd2-490d-95ce-d13232dc4eb2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 534.944556] env[62736]: DEBUG nova.compute.manager [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Refreshing instance network info cache due to event network-changed-fdbabc59-acd2-490d-95ce-d13232dc4eb2. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 534.944777] env[62736]: DEBUG oslo_concurrency.lockutils [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] Acquiring lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 534.944911] env[62736]: DEBUG oslo_concurrency.lockutils [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] Acquired lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 534.945059] env[62736]: DEBUG nova.network.neutron [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Refreshing network info cache for port fdbabc59-acd2-490d-95ce-d13232dc4eb2 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 534.945504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] Expecting reply to msg 21e431ca73654a5fbccbb0ceadb1e2b1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 534.958805] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21e431ca73654a5fbccbb0ceadb1e2b1 [ 535.053187] env[62736]: DEBUG nova.network.neutron [-] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 535.053662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 97943e2374e34acc9a3b78659592fd77 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.064089] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97943e2374e34acc9a3b78659592fd77 [ 535.085101] env[62736]: DEBUG nova.network.neutron [-] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 535.085614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6f4996012e614b59923adbec2f5d058c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.092928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f4996012e614b59923adbec2f5d058c [ 535.104410] env[62736]: DEBUG nova.compute.manager [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Received event network-changed-a989424f-fd97-4857-9982-210001ac2adc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 535.104604] env[62736]: DEBUG nova.compute.manager [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Refreshing instance network info cache due to event network-changed-a989424f-fd97-4857-9982-210001ac2adc. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 535.104862] env[62736]: DEBUG oslo_concurrency.lockutils [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] Acquiring lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 535.104959] env[62736]: DEBUG oslo_concurrency.lockutils [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] Acquired lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 535.105095] env[62736]: DEBUG nova.network.neutron [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Refreshing network info cache for port a989424f-fd97-4857-9982-210001ac2adc {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 535.105473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] Expecting reply to msg bbca2d9243924e149c8d05612462645d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.111920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbca2d9243924e149c8d05612462645d [ 535.271092] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 34697d8910574e64b8e6126ad7009165 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.272784] env[62736]: DEBUG nova.compute.utils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 535.274113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg c7ec4552347a4365a7240d0fcc81ebc0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.274208] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 535.274372] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 535.279833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 34697d8910574e64b8e6126ad7009165 [ 535.290540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7ec4552347a4365a7240d0fcc81ebc0 [ 535.395516] env[62736]: DEBUG nova.policy [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '418f4c924f4d45e69a85395d28c6f8b8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0c8a7ccd195e42f2a29370a2f89c9c67', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 535.476838] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.477146] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.477619] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 535.477619] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 535.478262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 5ad45e8727e349a3a7e3bfafa27ef239 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.487033] env[62736]: DEBUG nova.network.neutron [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 535.492853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ad45e8727e349a3a7e3bfafa27ef239 [ 535.555833] env[62736]: INFO nova.compute.manager [-] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Took 1.03 seconds to deallocate network for instance. [ 535.558389] env[62736]: DEBUG nova.compute.claims [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 535.558565] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.587068] env[62736]: DEBUG nova.network.neutron [-] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 535.587517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b402f26051754ec385ab7c60289f1b0f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.595234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b402f26051754ec385ab7c60289f1b0f [ 535.652505] env[62736]: DEBUG nova.network.neutron [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 535.669138] env[62736]: DEBUG nova.network.neutron [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 535.669712] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] Expecting reply to msg 14ec368cd7ca4c12908b0fc24bc37741 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.679769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14ec368cd7ca4c12908b0fc24bc37741 [ 535.779590] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 535.781321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg a76fa54f998e4975b0b376014913e275 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 535.829686] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a76fa54f998e4975b0b376014913e275 [ 535.986008] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 535.986139] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 535.986296] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 535.986400] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 535.986495] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 535.986612] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 535.989352] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.989777] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.989982] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.990164] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.990708] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.990708] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.990856] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 535.990912] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 535.991253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg edaa8a3c5425403ea439c7ab23c46b02 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 536.005187] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edaa8a3c5425403ea439c7ab23c46b02 [ 536.048883] env[62736]: DEBUG nova.network.neutron [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 536.049374] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] Expecting reply to msg b9f66342f2de4f1383780046a9d9d795 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 536.055625] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3349454-650d-4bff-b855-9f67788c22a6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.058633] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9f66342f2de4f1383780046a9d9d795 [ 536.064137] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d6255ab-0b45-4341-ba58-09521544e526 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.093438] env[62736]: INFO nova.compute.manager [-] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Took 1.18 seconds to deallocate network for instance. [ 536.095805] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d4c19a3-069e-4b0e-98f9-647010f0ff3b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.098731] env[62736]: DEBUG nova.compute.claims [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 536.098805] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 536.104471] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19042be8-df70-493f-b2c5-5b03febb62c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.122158] env[62736]: DEBUG nova.compute.provider_tree [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 536.122990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg a5cde9a56fff4908b56a32ed8ac2c733 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 536.133387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5cde9a56fff4908b56a32ed8ac2c733 [ 536.172409] env[62736]: DEBUG oslo_concurrency.lockutils [req-00a45697-7cae-4357-ae82-e5f2ce14c5d7 req-1fba3ecf-61da-4ecf-b970-8e4cb1ade613 service nova] Releasing lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 536.285541] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 606607d18ff84e5ebe09e2ecdb9ee884 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 536.337647] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 606607d18ff84e5ebe09e2ecdb9ee884 [ 536.493654] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 536.551678] env[62736]: DEBUG oslo_concurrency.lockutils [req-b3c882cc-cdbe-4e60-af31-95265cfeca1e req-00dd6608-65a3-4552-b491-5f0dbd476658 service nova] Releasing lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 536.629533] env[62736]: DEBUG nova.scheduler.client.report [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 536.629533] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 9b44723a1daa40debf21e52ed299c974 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 536.657527] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b44723a1daa40debf21e52ed299c974 [ 536.699185] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Successfully created port: f558dc42-a489-4138-8f8c-cfef1c9c730b {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 536.788164] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 536.823039] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 536.823274] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 536.823419] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 536.823591] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 536.823778] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 536.824063] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 536.824395] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 536.827757] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 536.827757] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 536.827757] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 536.827757] env[62736]: DEBUG nova.virt.hardware [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 536.827757] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-803e913a-bdf6-4728-9c6d-9fa1b1bac526 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.837575] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b44858b7-1fec-4e0e-92f4-24faa37b9e97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 536.922000] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquiring lock "52f8e827-3faf-47c1-b793-ccacc5a0bc4c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 536.922236] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "52f8e827-3faf-47c1-b793-ccacc5a0bc4c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 537.084694] env[62736]: ERROR nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 537.084694] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 537.084694] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 537.084694] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 537.084694] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 537.084694] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 537.084694] env[62736]: ERROR nova.compute.manager raise self.value [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 537.084694] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 537.084694] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 537.084694] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 537.085144] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 537.085144] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 537.085144] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 537.085144] env[62736]: ERROR nova.compute.manager [ 537.085144] env[62736]: Traceback (most recent call last): [ 537.085144] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 537.085144] env[62736]: listener.cb(fileno) [ 537.085144] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 537.085144] env[62736]: result = function(*args, **kwargs) [ 537.085144] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 537.085144] env[62736]: return func(*args, **kwargs) [ 537.085144] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 537.085144] env[62736]: raise e [ 537.085144] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 537.085144] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 537.085144] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 537.085144] env[62736]: created_port_ids = self._update_ports_for_instance( [ 537.085144] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 537.085144] env[62736]: with excutils.save_and_reraise_exception(): [ 537.085144] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 537.085144] env[62736]: self.force_reraise() [ 537.085144] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 537.085144] env[62736]: raise self.value [ 537.085144] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 537.085144] env[62736]: updated_port = self._update_port( [ 537.085144] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 537.085144] env[62736]: _ensure_no_port_binding_failure(port) [ 537.085144] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 537.085144] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 537.085873] env[62736]: nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 537.085873] env[62736]: Removing descriptor: 17 [ 537.085873] env[62736]: ERROR nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Traceback (most recent call last): [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] yield resources [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.driver.spawn(context, instance, image_meta, [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 537.085873] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] vm_ref = self.build_virtual_machine(instance, [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] vif_infos = vmwarevif.get_vif_info(self._session, [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] for vif in network_info: [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self._sync_wrapper(fn, *args, **kwargs) [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.wait() [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self[:] = self._gt.wait() [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self._exit_event.wait() [ 537.086166] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] result = hub.switch() [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self.greenlet.switch() [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] result = function(*args, **kwargs) [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return func(*args, **kwargs) [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise e [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] nwinfo = self.network_api.allocate_for_instance( [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 537.086517] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] created_port_ids = self._update_ports_for_instance( [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] with excutils.save_and_reraise_exception(): [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.force_reraise() [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise self.value [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] updated_port = self._update_port( [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] _ensure_no_port_binding_failure(port) [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 537.086884] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise exception.PortBindingFailed(port_id=port['id']) [ 537.087191] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 537.087191] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] [ 537.087191] env[62736]: INFO nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Terminating instance [ 537.088272] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquiring lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 537.088411] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquired lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 537.088567] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 537.089025] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 995b04c277a242a39f510dd561e34cc5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 537.113325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 995b04c277a242a39f510dd561e34cc5 [ 537.131198] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.374s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 537.131731] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 537.133451] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg e3dc983b38354acb8285ade18d8d6986 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 537.134636] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.396s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 537.136531] env[62736]: INFO nova.compute.claims [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 537.138230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 99cb7faddd4141509fcf82dd3523581e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 537.209651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3dc983b38354acb8285ade18d8d6986 [ 537.241131] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99cb7faddd4141509fcf82dd3523581e [ 537.641674] env[62736]: DEBUG nova.compute.utils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 537.642693] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg a5934a9ef2cc44db8b6f95f542d1ccf5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 537.645609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg ddc1859c067744f0902e0b0031541c77 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 537.646995] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 537.647272] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 537.655562] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 537.659405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5934a9ef2cc44db8b6f95f542d1ccf5 [ 537.660047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddc1859c067744f0902e0b0031541c77 [ 537.924056] env[62736]: DEBUG nova.policy [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '87f04a89778e4350a90bcc998a96de0f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '77a894bddd55484d956e0569e6d26d38', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 538.029903] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 538.030447] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 85d5db509f804190a1cc976bb0248e18 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.038890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85d5db509f804190a1cc976bb0248e18 [ 538.068724] env[62736]: DEBUG nova.compute.manager [req-327e5beb-7f60-436d-bd99-96d2dccb3e28 req-cfc818f6-1c62-4cea-a4e1-24c45b4fabea service nova] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Received event network-vif-deleted-fdbabc59-acd2-490d-95ce-d13232dc4eb2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 538.095313] env[62736]: DEBUG nova.compute.manager [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Received event network-vif-deleted-a989424f-fd97-4857-9982-210001ac2adc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 538.095313] env[62736]: DEBUG nova.compute.manager [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Received event network-changed-b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 538.095313] env[62736]: DEBUG nova.compute.manager [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Refreshing instance network info cache due to event network-changed-b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 538.095484] env[62736]: DEBUG oslo_concurrency.lockutils [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] Acquiring lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 538.147484] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 538.149205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 9f61cfb991d54f039632d40f20356297 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.201372] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f61cfb991d54f039632d40f20356297 [ 538.346250] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eb93d51d-55e9-4561-9ff0-e87c7335f58c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.354014] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f70da15f-ee96-42dd-ab6b-59bfcb402ce7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.386036] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3adcde7-6f34-443f-abd9-914036903afb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.393651] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-946eae6e-fb7e-43c3-948a-4e740b86eb2d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.406859] env[62736]: DEBUG nova.compute.provider_tree [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 538.407388] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 09eec2c5d552459cbdc7c458a7e5d2f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.415019] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09eec2c5d552459cbdc7c458a7e5d2f4 [ 538.532428] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Releasing lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 538.532874] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 538.533062] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 538.533364] env[62736]: DEBUG oslo_concurrency.lockutils [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] Acquired lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 538.533531] env[62736]: DEBUG nova.network.neutron [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Refreshing network info cache for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 538.533961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] Expecting reply to msg 5832119f9f22438393050f5102e8c2e2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.534736] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-06af5d25-223a-49a4-8edf-363915c27038 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.543732] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40d226e0-2cfa-4fe1-b600-813a286b9de9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 538.554071] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5832119f9f22438393050f5102e8c2e2 [ 538.565645] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 03b95ad5-85fe-41ad-987f-8140380f0a0d could not be found. [ 538.565892] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 538.566098] env[62736]: INFO nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 538.566344] env[62736]: DEBUG oslo.service.loopingcall [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 538.566719] env[62736]: DEBUG nova.compute.manager [-] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 538.566826] env[62736]: DEBUG nova.network.neutron [-] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 538.655053] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 038faea7796e4dd6852c94655ff7a637 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.656894] env[62736]: DEBUG nova.network.neutron [-] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 538.657547] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 645178daf25942b5b87ef67d7ee1809b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.678904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 645178daf25942b5b87ef67d7ee1809b [ 538.679643] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquiring lock "f537a3a9-b63a-4233-968a-942aca6ada0e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 538.679900] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "f537a3a9-b63a-4233-968a-942aca6ada0e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 538.694740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 038faea7796e4dd6852c94655ff7a637 [ 538.910527] env[62736]: DEBUG nova.scheduler.client.report [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 538.913030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg ec40e4a4163f4b0f978f3c24f3f97bd1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 538.926540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec40e4a4163f4b0f978f3c24f3f97bd1 [ 539.116944] env[62736]: DEBUG nova.network.neutron [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 539.160351] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 539.162898] env[62736]: DEBUG nova.network.neutron [-] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 539.163288] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 31b307f9d9fd42dd81fbdeeaf8c9a900 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.173802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31b307f9d9fd42dd81fbdeeaf8c9a900 [ 539.186284] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 539.186520] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 539.186685] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 539.186842] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 539.186984] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 539.187124] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 539.187322] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 539.187474] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 539.187632] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 539.187783] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 539.187946] env[62736]: DEBUG nova.virt.hardware [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 539.188874] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ba77750-539a-4d4a-8a48-a6828df0aeae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 539.198736] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de46798c-17c1-458c-90ce-7137c9b060a0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 539.415804] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.281s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 539.416409] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 539.418105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 0f50e22dabb84189b8449beea5cae922 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.419103] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 9.513s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 539.420587] env[62736]: INFO nova.compute.claims [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 539.422124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 3b9d3b0d2a3f4c8890f392233dfc0658 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.463157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f50e22dabb84189b8449beea5cae922 [ 539.470428] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b9d3b0d2a3f4c8890f392233dfc0658 [ 539.534180] env[62736]: DEBUG nova.network.neutron [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 539.534611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] Expecting reply to msg 1e3eb871bfef4cb2b6b0ce4e4bbfe9e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.544730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e3eb871bfef4cb2b6b0ce4e4bbfe9e5 [ 539.592042] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Successfully created port: d7e16639-ee52-48ff-bbd6-ffe2b267f1d7 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 539.665226] env[62736]: INFO nova.compute.manager [-] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Took 1.10 seconds to deallocate network for instance. [ 539.667722] env[62736]: DEBUG nova.compute.claims [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 539.667915] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 539.934345] env[62736]: DEBUG nova.compute.utils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 539.935101] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 730d686f0e44413f8c514c0c623e94be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.937062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg e4f01d008d4d423ead187cfbb4c101c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 539.937989] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 539.948211] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4f01d008d4d423ead187cfbb4c101c5 [ 539.948749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 730d686f0e44413f8c514c0c623e94be [ 540.036847] env[62736]: DEBUG oslo_concurrency.lockutils [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] Releasing lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 540.037119] env[62736]: DEBUG nova.compute.manager [req-7800cc9b-86e3-4b79-82b3-5b5e591bea1d req-8c09c227-788b-4421-971c-388b28e6d933 service nova] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Received event network-vif-deleted-b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 540.448047] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 540.448047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 8338e24336854a7c834bd28b083d4144 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 540.501052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8338e24336854a7c834bd28b083d4144 [ 540.678557] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87add0c3-6832-4e09-9f52-4c871caa6094 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 540.686971] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fde0d3f0-2d2f-4941-9e04-21cf7755c9ec {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 540.717326] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98c38b8f-d50d-464e-a05b-b6b675e4c151 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 540.725463] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74b52d43-3f89-4854-9d61-f6c872e25ca7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 540.740353] env[62736]: DEBUG nova.compute.provider_tree [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 540.740853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg cca7641a8fc841839c70a37fbd1a9ff9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 540.749498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cca7641a8fc841839c70a37fbd1a9ff9 [ 540.958507] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg b701d9f7f7dd4e30b51d05315abf0a55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.010316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b701d9f7f7dd4e30b51d05315abf0a55 [ 541.214203] env[62736]: ERROR nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 541.214203] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.214203] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 541.214203] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 541.214203] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.214203] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.214203] env[62736]: ERROR nova.compute.manager raise self.value [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 541.214203] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 541.214203] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.214203] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 541.214715] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.214715] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 541.214715] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 541.214715] env[62736]: ERROR nova.compute.manager [ 541.214715] env[62736]: Traceback (most recent call last): [ 541.214715] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 541.214715] env[62736]: listener.cb(fileno) [ 541.214715] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 541.214715] env[62736]: result = function(*args, **kwargs) [ 541.214715] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 541.214715] env[62736]: return func(*args, **kwargs) [ 541.214715] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 541.214715] env[62736]: raise e [ 541.214715] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.214715] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 541.214715] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 541.214715] env[62736]: created_port_ids = self._update_ports_for_instance( [ 541.214715] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 541.214715] env[62736]: with excutils.save_and_reraise_exception(): [ 541.214715] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.214715] env[62736]: self.force_reraise() [ 541.214715] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.214715] env[62736]: raise self.value [ 541.214715] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 541.214715] env[62736]: updated_port = self._update_port( [ 541.214715] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.214715] env[62736]: _ensure_no_port_binding_failure(port) [ 541.214715] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.214715] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 541.215774] env[62736]: nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 541.215774] env[62736]: Removing descriptor: 15 [ 541.215774] env[62736]: ERROR nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Traceback (most recent call last): [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] yield resources [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.driver.spawn(context, instance, image_meta, [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 541.215774] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] vm_ref = self.build_virtual_machine(instance, [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] for vif in network_info: [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self._sync_wrapper(fn, *args, **kwargs) [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.wait() [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self[:] = self._gt.wait() [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self._exit_event.wait() [ 541.216265] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] result = hub.switch() [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self.greenlet.switch() [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] result = function(*args, **kwargs) [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return func(*args, **kwargs) [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise e [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] nwinfo = self.network_api.allocate_for_instance( [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 541.216635] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] created_port_ids = self._update_ports_for_instance( [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] with excutils.save_and_reraise_exception(): [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.force_reraise() [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise self.value [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] updated_port = self._update_port( [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] _ensure_no_port_binding_failure(port) [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 541.217045] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise exception.PortBindingFailed(port_id=port['id']) [ 541.217395] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 541.217395] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] [ 541.217395] env[62736]: INFO nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Terminating instance [ 541.217492] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquiring lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 541.217597] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquired lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 541.217753] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 541.218164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 91adb093857b414ea527f242bbf3437e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.227272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91adb093857b414ea527f242bbf3437e [ 541.243452] env[62736]: DEBUG nova.scheduler.client.report [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 541.245774] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 4e312b1d208a4cec99fe8da0ff929bd0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.258191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e312b1d208a4cec99fe8da0ff929bd0 [ 541.462122] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 541.489573] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 541.489825] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 541.489976] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 541.490151] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 541.490328] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 541.490480] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 541.490681] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 541.490831] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 541.490988] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 541.491138] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 541.491301] env[62736]: DEBUG nova.virt.hardware [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 541.492184] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3e725bf-3fd1-4849-af83-c4f9bb5dbd87 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.501030] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbc41d03-349f-4728-9507-2e8ca35d3f95 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.518109] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 541.534266] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 541.534599] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c0ee1cac-61ff-4092-859f-35a2492a693e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.553072] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Created folder: OpenStack in parent group-v4. [ 541.553231] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating folder: Project (1df47f750ca24959bc88e535e1c8f075). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 541.553463] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-36f9a574-f3fc-4c39-b6fe-18a3b4d8c17a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.566616] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Created folder: Project (1df47f750ca24959bc88e535e1c8f075) in parent group-v107913. [ 541.566890] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating folder: Instances. Parent ref: group-v107914. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 541.567024] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d72d0596-4b68-4446-9322-0645968e4459 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.577460] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Created folder: Instances in parent group-v107914. [ 541.577971] env[62736]: DEBUG oslo.service.loopingcall [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 541.578127] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 541.578283] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2fd0bd14-e6f2-47db-ad4e-3cea9dd13d4f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 541.598327] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 541.598327] env[62736]: value = "task-397656" [ 541.598327] env[62736]: _type = "Task" [ 541.598327] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 541.613278] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397656, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 541.756023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.331s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.756023] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 541.756023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg a12f9569da544003a65e03d07418a7aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.756023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.117s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.756023] env[62736]: INFO nova.compute.claims [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 541.757604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 8b7d5954155547658f56da100eb86fbf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.766944] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 541.819253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b7d5954155547658f56da100eb86fbf [ 541.823826] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a12f9569da544003a65e03d07418a7aa [ 541.946935] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 541.947518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 56a85441d03f456398808bd2dd71534f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 541.957531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56a85441d03f456398808bd2dd71534f [ 542.080397] env[62736]: DEBUG nova.compute.manager [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Received event network-changed-f558dc42-a489-4138-8f8c-cfef1c9c730b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 542.080607] env[62736]: DEBUG nova.compute.manager [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Refreshing instance network info cache due to event network-changed-f558dc42-a489-4138-8f8c-cfef1c9c730b. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 542.080811] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] Acquiring lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 542.116737] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397656, 'name': CreateVM_Task, 'duration_secs': 0.262257} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 542.117043] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 542.118373] env[62736]: DEBUG oslo_vmware.service [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34074da7-3d75-42c6-a609-c56c45845c78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.125123] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 542.125333] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 542.126005] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 542.126254] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-08fef9a7-3595-41a5-9427-f71ef964f4a9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.133922] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 542.133922] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52acdeb9-f484-7414-1ac9-cdf969b7eacc" [ 542.133922] env[62736]: _type = "Task" [ 542.133922] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 542.142493] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52acdeb9-f484-7414-1ac9-cdf969b7eacc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 542.223423] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquiring lock "bd41759b-5c32-42e6-b625-4ea48f98045e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 542.223842] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "bd41759b-5c32-42e6-b625-4ea48f98045e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 542.260141] env[62736]: DEBUG nova.compute.utils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 542.260983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg f57d48e457444e25a62e88a25dd544d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.261915] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 542.262126] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 542.265319] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg b90932e8b55a4a0cbbf82488e06eee67 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.277052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b90932e8b55a4a0cbbf82488e06eee67 [ 542.281935] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f57d48e457444e25a62e88a25dd544d2 [ 542.453076] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Releasing lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 542.453076] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 542.453076] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 542.453076] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] Acquired lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 542.453076] env[62736]: DEBUG nova.network.neutron [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Refreshing network info cache for port f558dc42-a489-4138-8f8c-cfef1c9c730b {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 542.453273] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] Expecting reply to msg 5b0314b274fc442fae087c4a3b2a6e64 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.453273] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-30113b4a-03bd-4af7-814f-4274f4f84ac5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.457259] env[62736]: DEBUG nova.policy [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a145ce1c57cd43379c748e2731e3865c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6ce3b399d3c649618af0af533baaf84b', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 542.459610] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b0314b274fc442fae087c4a3b2a6e64 [ 542.465194] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c179cc22-b58d-4f21-90b0-b3765c1a7955 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.496468] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2 could not be found. [ 542.496468] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 542.496468] env[62736]: INFO nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Took 0.05 seconds to destroy the instance on the hypervisor. [ 542.496468] env[62736]: DEBUG oslo.service.loopingcall [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 542.501012] env[62736]: DEBUG nova.compute.manager [-] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 542.501283] env[62736]: DEBUG nova.network.neutron [-] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 542.588553] env[62736]: DEBUG nova.network.neutron [-] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 542.588553] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 66fa025977b7483fb0c57baa90de0568 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.602671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66fa025977b7483fb0c57baa90de0568 [ 542.645233] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 542.645580] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 542.648312] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 542.648473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 542.648946] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 542.649272] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6a867a77-189c-41df-a020-2b3be2441cb1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.672121] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 542.672121] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 542.672121] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a22febf-86c6-49a5-b633-0edba8805c9f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.678361] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a09c2673-9892-4b24-b65d-1090040ff0b5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.683425] env[62736]: ERROR nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 542.683425] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 542.683425] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 542.683425] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 542.683425] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 542.683425] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 542.683425] env[62736]: ERROR nova.compute.manager raise self.value [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 542.683425] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 542.683425] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 542.683425] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 542.683944] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 542.683944] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 542.683944] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 542.683944] env[62736]: ERROR nova.compute.manager [ 542.684464] env[62736]: Traceback (most recent call last): [ 542.684548] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 542.684548] env[62736]: listener.cb(fileno) [ 542.684618] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 542.684618] env[62736]: result = function(*args, **kwargs) [ 542.684736] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 542.684736] env[62736]: return func(*args, **kwargs) [ 542.684801] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 542.684801] env[62736]: raise e [ 542.684874] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 542.684874] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 542.684943] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 542.684943] env[62736]: created_port_ids = self._update_ports_for_instance( [ 542.685009] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 542.685009] env[62736]: with excutils.save_and_reraise_exception(): [ 542.685092] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 542.685092] env[62736]: self.force_reraise() [ 542.685156] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 542.685156] env[62736]: raise self.value [ 542.685218] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 542.685218] env[62736]: updated_port = self._update_port( [ 542.685292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 542.685292] env[62736]: _ensure_no_port_binding_failure(port) [ 542.685356] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 542.685356] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 542.685440] env[62736]: nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 542.685486] env[62736]: Removing descriptor: 19 [ 542.686290] env[62736]: ERROR nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Traceback (most recent call last): [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] yield resources [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.driver.spawn(context, instance, image_meta, [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self._vmops.spawn(context, instance, image_meta, injected_files, [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] vm_ref = self.build_virtual_machine(instance, [ 542.686290] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] vif_infos = vmwarevif.get_vif_info(self._session, [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] for vif in network_info: [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self._sync_wrapper(fn, *args, **kwargs) [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.wait() [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self[:] = self._gt.wait() [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self._exit_event.wait() [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 542.686653] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] result = hub.switch() [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self.greenlet.switch() [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] result = function(*args, **kwargs) [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return func(*args, **kwargs) [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise e [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] nwinfo = self.network_api.allocate_for_instance( [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] created_port_ids = self._update_ports_for_instance( [ 542.687104] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] with excutils.save_and_reraise_exception(): [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.force_reraise() [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise self.value [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] updated_port = self._update_port( [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] _ensure_no_port_binding_failure(port) [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise exception.PortBindingFailed(port_id=port['id']) [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 542.687528] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] [ 542.688256] env[62736]: INFO nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Terminating instance [ 542.695170] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 542.695170] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]529a621c-b865-b111-61f7-cb176fedcb3b" [ 542.695170] env[62736]: _type = "Task" [ 542.695170] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 542.697143] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 542.697608] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquired lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 542.698012] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 542.698736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 9d4910f80a3146a69d9a58e3c728cddb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.706545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d4910f80a3146a69d9a58e3c728cddb [ 542.710765] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Preparing fetch location {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 542.711037] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating directory with path [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 542.711249] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-12f6aa52-5029-49ec-8235-3273aac9d4a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.734134] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Created directory with path [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 542.734305] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Fetch image to [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 542.734454] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Downloading image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk on the data store datastore1 {{(pid=62736) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 542.735189] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbcfdad1-3d5f-4c00-9682-02b44c101887 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.752657] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5c645d1-af2c-40a2-a895-e380cd9e4430 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.769480] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 542.771267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg db6a27cfded64731bec892f23f18faa9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 542.780946] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56343fc1-389b-44c4-8440-896266e933c7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.831608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db6a27cfded64731bec892f23f18faa9 [ 542.835109] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-697a41d3-8053-4867-a671-8a4bb3e5c976 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.842654] env[62736]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-1f15761e-484e-464a-a676-ca8f5b6e4148 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 542.939062] env[62736]: DEBUG nova.virt.vmwareapi.images [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Downloading image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to the data store datastore1 {{(pid=62736) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 542.984633] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquiring lock "25508ff0-bc62-47b3-ae9e-7d9d8c735edc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 542.984875] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "25508ff0-bc62-47b3-ae9e-7d9d8c735edc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.030386] env[62736]: DEBUG oslo_vmware.rw_handles [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=62736) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 543.092369] env[62736]: DEBUG nova.network.neutron [-] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 543.092830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 766285ef9b4648a08522f2b9f0738759 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.096537] env[62736]: DEBUG nova.network.neutron [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 543.103614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 766285ef9b4648a08522f2b9f0738759 [ 543.162513] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d687026-0d20-4540-923d-8763a99583b2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.187867] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d124a405-74ab-49b1-baea-007afdb1b874 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.240065] env[62736]: DEBUG nova.network.neutron [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 543.240626] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] Expecting reply to msg 9f379fbad2ad48e9bd7d08ff668670ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.248907] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01cfa086-6551-4922-aacc-7e27c95e47ae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.252932] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f379fbad2ad48e9bd7d08ff668670ca [ 543.266458] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4239b175-b49a-4f7d-9030-2013ff056a0a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.286837] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 3d0cc8926d3a4005a6b20614f84f87d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.298364] env[62736]: DEBUG nova.compute.provider_tree [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 543.299344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg ff2e48b02cba41a8958a8193a42dd959 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.301829] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 543.317067] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff2e48b02cba41a8958a8193a42dd959 [ 543.329580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d0cc8926d3a4005a6b20614f84f87d0 [ 543.439925] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 543.440379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg bf13120ad4384880a3e7eb998e82c030 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.451655] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf13120ad4384880a3e7eb998e82c030 [ 543.599467] env[62736]: INFO nova.compute.manager [-] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Took 1.10 seconds to deallocate network for instance. [ 543.602686] env[62736]: DEBUG nova.compute.claims [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 543.602922] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.625840] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Successfully created port: 3051a2c4-e501-45c9-b625-6fcae110ce27 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 543.754174] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2c47d03-11a8-4d69-b6e5-77dd040a041b req-49909be0-09a4-4e8f-bf15-e93c25a47d53 service nova] Releasing lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 543.778677] env[62736]: DEBUG oslo_vmware.rw_handles [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Completed reading data from the image iterator. {{(pid=62736) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 543.778986] env[62736]: DEBUG oslo_vmware.rw_handles [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=62736) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 543.801528] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 543.810059] env[62736]: DEBUG nova.scheduler.client.report [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 543.812330] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg ebd2c130d3794a92bfde409668658a8d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 543.825520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebd2c130d3794a92bfde409668658a8d [ 543.834248] env[62736]: DEBUG nova.virt.vmwareapi.images [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Downloaded image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk on the data store datastore1 {{(pid=62736) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 543.836717] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Caching image {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 543.836985] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Copying Virtual Disk [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk to [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 543.837583] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8e49391c-cc70-4858-8fad-1d79bb774f9b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.842532] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 543.843364] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 543.843364] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 543.843364] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 543.843364] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 543.843584] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 543.843659] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 543.843785] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 543.843939] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 543.844280] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 543.844466] env[62736]: DEBUG nova.virt.hardware [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 543.845307] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d839d6c8-cc0e-4782-9a47-361e209236ed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.854874] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6e93db9-008b-4949-aa0c-9e1e84c40b6c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.859533] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 543.859533] env[62736]: value = "task-397657" [ 543.859533] env[62736]: _type = "Task" [ 543.859533] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 543.879213] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397657, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 543.948913] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Releasing lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 543.948913] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 543.949068] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 543.949355] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5e7cc780-6da0-4298-ab0d-7acb1fcc6692 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.957870] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ac14d84-c47c-4a74-834c-e02cc7c9361a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 543.981018] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 2eb46a2a-27fc-4d44-9d18-6d4553026a60 could not be found. [ 543.981191] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 543.981379] env[62736]: INFO nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Took 0.03 seconds to destroy the instance on the hypervisor. [ 543.981613] env[62736]: DEBUG oslo.service.loopingcall [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 543.981891] env[62736]: DEBUG nova.compute.manager [-] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 543.981928] env[62736]: DEBUG nova.network.neutron [-] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 544.048553] env[62736]: DEBUG nova.network.neutron [-] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 544.049306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d321edcaf0fd4b07ab55c0bbde167190 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.056671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d321edcaf0fd4b07ab55c0bbde167190 [ 544.317325] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.564s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 544.317911] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 544.319698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 50d4fc7b5fd04e3a971ec4f98ebb204e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.321532] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.752s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 544.323052] env[62736]: INFO nova.compute.claims [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 544.325746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg dd8f4b51bab24a5693709d03320491e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.369743] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397657, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 544.375509] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50d4fc7b5fd04e3a971ec4f98ebb204e [ 544.377198] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd8f4b51bab24a5693709d03320491e1 [ 544.552744] env[62736]: DEBUG nova.network.neutron [-] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 544.552744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b0014cde990d41e28b71db356f9ee0a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.561896] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0014cde990d41e28b71db356f9ee0a0 [ 544.829036] env[62736]: DEBUG nova.compute.utils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 544.829715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 8dabe68817e0495eb605f44340deef6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.831869] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 69738eced6ed48b38d132978e9b5b66e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 544.833468] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 544.833468] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 544.841425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dabe68817e0495eb605f44340deef6a [ 544.843393] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69738eced6ed48b38d132978e9b5b66e [ 544.869678] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397657, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.669699} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 544.869959] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Copied Virtual Disk [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk to [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 544.870134] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleting the datastore file [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 544.870389] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c3cd6f6b-7a32-47f3-8fa8-e46ef4c4138e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 544.878973] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 544.878973] env[62736]: value = "task-397658" [ 544.878973] env[62736]: _type = "Task" [ 544.878973] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 544.885181] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397658, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 545.054707] env[62736]: INFO nova.compute.manager [-] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Took 1.07 seconds to deallocate network for instance. [ 545.058378] env[62736]: DEBUG nova.compute.claims [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 545.058523] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.159649] env[62736]: DEBUG nova.policy [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a9a0e0bbae8046a783b171c8a4ee3c11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f9f409d646e3429cac4cd3555a5073c2', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 545.338678] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 545.340627] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 768f49ba32904eb79ea5fc436e3f9ec5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 545.384433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 768f49ba32904eb79ea5fc436e3f9ec5 [ 545.392884] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397658, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.088164} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 545.392971] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 545.393119] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Moving file from [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4/81867c62-ef8e-483f-bfd2-854abdcd6db5 to [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5. {{(pid=62736) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 545.393358] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-1d16991a-70fd-4dd9-8da7-16af1b8f5de7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.400584] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 545.400584] env[62736]: value = "task-397659" [ 545.400584] env[62736]: _type = "Task" [ 545.400584] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 545.408336] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397659, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 545.589651] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8a6d355-6af4-4613-b0ef-ae7e782020d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.598491] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24e3b97e-e156-48c5-9a74-fd35fbf41775 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.647115] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bad2de8-b6f4-4967-9b52-a9bb586fb8b9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.654751] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ea3d2a9-fe06-4af9-96f0-f6c457df203d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.669111] env[62736]: DEBUG nova.compute.provider_tree [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 545.669672] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 8cf11e44133d4e26a280b23c815b64e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 545.676920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8cf11e44133d4e26a280b23c815b64e9 [ 545.847544] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg bf173c906baf4a22b621701f1d517b8e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 545.888951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf173c906baf4a22b621701f1d517b8e [ 545.910576] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397659, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.029002} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 545.910853] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] File moved {{(pid=62736) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 545.911043] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Cleaning up location [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 545.911198] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleting the datastore file [datastore1] vmware_temp/418dfbfe-d44c-4237-9afe-02e0707eefd4 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 545.911436] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-690b3151-915a-43e6-b9c3-4132618285b4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 545.918295] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 545.918295] env[62736]: value = "task-397660" [ 545.918295] env[62736]: _type = "Task" [ 545.918295] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 545.926586] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397660, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 545.993916] env[62736]: DEBUG nova.compute.manager [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Received event network-vif-deleted-f558dc42-a489-4138-8f8c-cfef1c9c730b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 545.993916] env[62736]: DEBUG nova.compute.manager [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Received event network-changed-d7e16639-ee52-48ff-bbd6-ffe2b267f1d7 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 545.994045] env[62736]: DEBUG nova.compute.manager [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Refreshing instance network info cache due to event network-changed-d7e16639-ee52-48ff-bbd6-ffe2b267f1d7. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 545.994695] env[62736]: DEBUG oslo_concurrency.lockutils [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] Acquiring lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 545.994695] env[62736]: DEBUG oslo_concurrency.lockutils [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] Acquired lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 545.994695] env[62736]: DEBUG nova.network.neutron [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Refreshing network info cache for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 545.994913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] Expecting reply to msg 96ea6e2df6b142ff9c852b337e75128d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 546.002052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96ea6e2df6b142ff9c852b337e75128d [ 546.166327] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Successfully created port: 374282b5-a930-4685-a3e9-c381d14b89c8 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 546.172286] env[62736]: DEBUG nova.scheduler.client.report [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 546.174616] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg abe5dc4b2705497bb9e31e299f78798d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 546.191234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abe5dc4b2705497bb9e31e299f78798d [ 546.351802] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 546.377813] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 546.377979] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 546.378123] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 546.378326] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 546.378511] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 546.378687] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 546.379003] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 546.379259] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 546.379574] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 546.379797] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 546.380194] env[62736]: DEBUG nova.virt.hardware [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 546.382109] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19d0ca71-b426-425d-8ae8-3acc9437eaf0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.390597] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e64d738f-9ccd-4120-b979-5a6206138f3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.426427] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397660, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.024995} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 546.426660] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 546.427367] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ed519264-589b-445e-8e1b-7f8ce27e416d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.432243] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 546.432243] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]522e999e-fff3-5d53-044b-79dea0126c60" [ 546.432243] env[62736]: _type = "Task" [ 546.432243] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 546.439552] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]522e999e-fff3-5d53-044b-79dea0126c60, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 546.648980] env[62736]: DEBUG nova.network.neutron [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 546.684510] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.361s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 546.684510] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 546.684510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 7236109f766944e981157277d5d76d28 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 546.685310] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.222s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 546.686793] env[62736]: INFO nova.compute.claims [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 546.688404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 08b8f6519f904988937a6a792913ddab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 546.737442] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7236109f766944e981157277d5d76d28 [ 546.748354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08b8f6519f904988937a6a792913ddab [ 546.914321] env[62736]: DEBUG nova.network.neutron [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 546.914321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] Expecting reply to msg 5d12d6333bc54da1a58124f0a15e01b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 546.924900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d12d6333bc54da1a58124f0a15e01b4 [ 546.943516] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]522e999e-fff3-5d53-044b-79dea0126c60, 'name': SearchDatastore_Task, 'duration_secs': 0.008649} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 546.943787] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 546.944071] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91/5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 546.944303] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-8db9b710-79f7-47bc-a081-1869903dd660 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 546.951645] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 546.951645] env[62736]: value = "task-397661" [ 546.951645] env[62736]: _type = "Task" [ 546.951645] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 546.960741] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397661, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 547.089701] env[62736]: ERROR nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 547.089701] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 547.089701] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 547.089701] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 547.089701] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 547.089701] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 547.089701] env[62736]: ERROR nova.compute.manager raise self.value [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 547.089701] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 547.089701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 547.089701] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 547.090182] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 547.090182] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 547.090182] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 547.090182] env[62736]: ERROR nova.compute.manager [ 547.090182] env[62736]: Traceback (most recent call last): [ 547.090182] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 547.090182] env[62736]: listener.cb(fileno) [ 547.090182] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 547.090182] env[62736]: result = function(*args, **kwargs) [ 547.090182] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 547.090182] env[62736]: return func(*args, **kwargs) [ 547.090182] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 547.090182] env[62736]: raise e [ 547.090182] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 547.090182] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 547.090182] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 547.090182] env[62736]: created_port_ids = self._update_ports_for_instance( [ 547.090182] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 547.090182] env[62736]: with excutils.save_and_reraise_exception(): [ 547.090182] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 547.090182] env[62736]: self.force_reraise() [ 547.090182] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 547.090182] env[62736]: raise self.value [ 547.090182] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 547.090182] env[62736]: updated_port = self._update_port( [ 547.090182] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 547.090182] env[62736]: _ensure_no_port_binding_failure(port) [ 547.090182] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 547.090182] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 547.090997] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 547.090997] env[62736]: Removing descriptor: 15 [ 547.091490] env[62736]: ERROR nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] Traceback (most recent call last): [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] yield resources [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.driver.spawn(context, instance, image_meta, [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self._vmops.spawn(context, instance, image_meta, injected_files, [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] vm_ref = self.build_virtual_machine(instance, [ 547.091490] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] vif_infos = vmwarevif.get_vif_info(self._session, [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] for vif in network_info: [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self._sync_wrapper(fn, *args, **kwargs) [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.wait() [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self[:] = self._gt.wait() [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self._exit_event.wait() [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 547.091846] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] result = hub.switch() [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self.greenlet.switch() [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] result = function(*args, **kwargs) [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return func(*args, **kwargs) [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise e [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] nwinfo = self.network_api.allocate_for_instance( [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] created_port_ids = self._update_ports_for_instance( [ 547.092211] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] with excutils.save_and_reraise_exception(): [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.force_reraise() [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise self.value [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] updated_port = self._update_port( [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] _ensure_no_port_binding_failure(port) [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise exception.PortBindingFailed(port_id=port['id']) [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 547.092551] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] [ 547.093037] env[62736]: INFO nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Terminating instance [ 547.095500] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquiring lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 547.095818] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquired lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 547.096123] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 547.097056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 2183591753e84f169b303fc317482c35 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 547.105949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2183591753e84f169b303fc317482c35 [ 547.192977] env[62736]: DEBUG nova.compute.utils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 547.193991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg bbb082691e5042e7abb12dc8aa49c826 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 547.197300] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg a288ed5bf2c24d4989fa9924378b677b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 547.204220] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 547.204220] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 547.206362] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbb082691e5042e7abb12dc8aa49c826 [ 547.206625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a288ed5bf2c24d4989fa9924378b677b [ 547.388678] env[62736]: DEBUG nova.policy [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 547.416274] env[62736]: DEBUG oslo_concurrency.lockutils [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] Releasing lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 547.417057] env[62736]: DEBUG nova.compute.manager [req-b34336ad-db0d-4b9a-b149-6139b9c57369 req-7d94613d-9dd9-4992-aef9-046d2f42b7ce service nova] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Received event network-vif-deleted-d7e16639-ee52-48ff-bbd6-ffe2b267f1d7 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 547.462196] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397661, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.489257} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 547.462661] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91/5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 547.463067] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 547.463479] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-d62c4c07-9eb7-4f03-b068-c8b484393fb6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 547.472924] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 547.472924] env[62736]: value = "task-397662" [ 547.472924] env[62736]: _type = "Task" [ 547.472924] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 547.480936] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397662, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 547.660767] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 547.704749] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 547.706492] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a7f8a65b2e664617b707a381c7704317 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 547.795214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7f8a65b2e664617b707a381c7704317 [ 547.984839] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397662, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.136996} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 547.984839] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 547.985724] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8857509d-41b6-485a-84a7-852b50f877f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.008187] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Reconfiguring VM instance instance-00000006 to attach disk [datastore1] 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91/5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 548.009485] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-fdae4d20-7acf-485c-8335-5830f5dc6efa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.024357] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-814b5e0c-0fad-4ab4-9b80-ebed7fc1be64 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.031623] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7312211-f9ea-4b18-a4b7-8ba3f642eedd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.035705] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 548.035705] env[62736]: value = "task-397663" [ 548.035705] env[62736]: _type = "Task" [ 548.035705] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 548.066823] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acb7ed7b-c4ae-4e0d-b107-bf17cb54e8fa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.070853] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397663, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 548.075447] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c0dbd9f-62ff-47d2-8269-4f527191972f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.088251] env[62736]: DEBUG nova.compute.provider_tree [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 548.088773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg ba135178ae984fada724898cb3ba9374 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 548.097034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba135178ae984fada724898cb3ba9374 [ 548.132129] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 548.132694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg e1864b762f9b4336a880d976090ad499 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 548.144327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1864b762f9b4336a880d976090ad499 [ 548.214604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg bb604293403f43d997e758cddc04ae9c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 548.289254] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquiring lock "918aa256-a10d-46d1-b9a4-ecd70cf3566f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 548.289254] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "918aa256-a10d-46d1-b9a4-ecd70cf3566f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 548.310328] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb604293403f43d997e758cddc04ae9c [ 548.546852] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397663, 'name': ReconfigVM_Task, 'duration_secs': 0.436382} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 548.547285] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Reconfigured VM instance instance-00000006 to attach disk [datastore1] 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91/5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 548.547996] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-d474f6b9-bc06-4627-a5fa-29f2fee320ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.556115] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 548.556115] env[62736]: value = "task-397668" [ 548.556115] env[62736]: _type = "Task" [ 548.556115] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 548.567602] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397668, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 548.593433] env[62736]: DEBUG nova.scheduler.client.report [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 548.595821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 347ec881786a42fd817bcc65fb3907be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 548.603535] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Successfully created port: ea932e45-f256-494c-9fc5-5ace73092f2f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 548.611261] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 347ec881786a42fd817bcc65fb3907be [ 548.635779] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Releasing lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 548.636425] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 548.636690] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 548.637202] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d5a0e826-85a0-468e-bf2f-433563d85121 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.646219] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a98b5d2-77e1-4542-b469-ad913c15b337 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.674886] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0b211be4-1955-456d-bf30-5195b5169807 could not be found. [ 548.675159] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 548.675424] env[62736]: INFO nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Took 0.04 seconds to destroy the instance on the hypervisor. [ 548.675719] env[62736]: DEBUG oslo.service.loopingcall [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 548.675959] env[62736]: DEBUG nova.compute.manager [-] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 548.676079] env[62736]: DEBUG nova.network.neutron [-] [instance: 0b211be4-1955-456d-bf30-5195b5169807] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 548.719023] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 548.730404] env[62736]: DEBUG nova.network.neutron [-] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 548.730404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b06892832a7e473695889f99fee46c84 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 548.744367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b06892832a7e473695889f99fee46c84 [ 548.792610] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 548.792876] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 548.793029] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 548.793231] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 548.793347] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 548.793496] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 548.793697] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 548.793845] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 548.794026] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 548.794183] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 548.794359] env[62736]: DEBUG nova.virt.hardware [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 548.795268] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc138c4a-ea3d-4748-bbca-dccdaf900cc3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 548.804060] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cf16757-fec9-4542-a265-738c0aa16c91 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.066038] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397668, 'name': Rename_Task, 'duration_secs': 0.134377} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 549.066315] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 549.066526] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-1efb037a-8b6c-4516-b1a8-fb096d3befb7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.073239] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 549.073239] env[62736]: value = "task-397669" [ 549.073239] env[62736]: _type = "Task" [ 549.073239] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 549.081753] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397669, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 549.104748] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.419s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 549.105274] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 549.107010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 50f09c3183af42bdb7c008b99cce63c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.108150] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.550s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.109998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg a3a73fd3beab4ad988209cb50998ed15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.160909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50f09c3183af42bdb7c008b99cce63c4 [ 549.175114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3a73fd3beab4ad988209cb50998ed15 [ 549.232168] env[62736]: DEBUG nova.network.neutron [-] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 549.232655] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 86b6da1f0d9e4d489130298c2329b602 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.241113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86b6da1f0d9e4d489130298c2329b602 [ 549.440333] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquiring lock "64ee370c-6d43-4e2b-a27a-3aa716fdc322" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.440987] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "64ee370c-6d43-4e2b-a27a-3aa716fdc322" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 549.582585] env[62736]: DEBUG oslo_vmware.api [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397669, 'name': PowerOnVM_Task, 'duration_secs': 0.466042} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 549.582845] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 549.583286] env[62736]: INFO nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Took 8.12 seconds to spawn the instance on the hypervisor. [ 549.583763] env[62736]: DEBUG nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 549.584946] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db64a7c1-623e-4424-a819-879304b77cf3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.593118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg fc71ea714a994c85ba556cc8cf35b8f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.612792] env[62736]: DEBUG nova.compute.utils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 549.613559] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 7ea804a06c154acb832a33d772bb5006 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.617646] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 549.618510] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 549.630316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc71ea714a994c85ba556cc8cf35b8f4 [ 549.638453] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ea804a06c154acb832a33d772bb5006 [ 549.735317] env[62736]: INFO nova.compute.manager [-] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Took 1.06 seconds to deallocate network for instance. [ 549.737812] env[62736]: DEBUG nova.compute.claims [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 549.738112] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 549.837963] env[62736]: DEBUG nova.policy [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '71886e19fbc749d989c55e17c178862a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7f30e3b24f414007b0066cab5346a3bf', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 549.865304] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b9877c7-ec02-46bd-83fd-315e6a17a5a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.875856] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88a83bfd-df7e-4ae2-b322-d2e2f1ca3d3d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.910183] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d8875ee-6afe-4ea7-b3b2-c165fbbfb3c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.920451] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85ea751a-f9e3-4d2c-93f7-3873342b086c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 549.935254] env[62736]: DEBUG nova.compute.provider_tree [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 549.935854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 2a9ff64df2284fba866f0c25cfb10ae9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 549.943396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a9ff64df2284fba866f0c25cfb10ae9 [ 550.103918] env[62736]: INFO nova.compute.manager [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Took 21.38 seconds to build instance. [ 550.104339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 0540c9533c6c4a7badfff07b71ba65eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.118266] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 550.122390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 7403028f07bd401380c4bbcd3dee8a6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.128623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0540c9533c6c4a7badfff07b71ba65eb [ 550.205371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7403028f07bd401380c4bbcd3dee8a6a [ 550.439072] env[62736]: DEBUG nova.scheduler.client.report [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 550.442414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg fa1d145654c549578a96eaddf8dd3c57 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.457653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa1d145654c549578a96eaddf8dd3c57 [ 550.606212] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6983c055-f4fc-40ff-af3d-b1c177f65e24 tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 22.891s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 550.606841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 8a16fa0c11674c0f997c237c2607daa6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.615998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a16fa0c11674c0f997c237c2607daa6 [ 550.629038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 03f0695376134c4b9b4602d6a667dced in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.668903] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03f0695376134c4b9b4602d6a667dced [ 550.945044] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.837s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 550.945732] env[62736]: ERROR nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Traceback (most recent call last): [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.driver.spawn(context, instance, image_meta, [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self._vmops.spawn(context, instance, image_meta, injected_files, [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] vm_ref = self.build_virtual_machine(instance, [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] vif_infos = vmwarevif.get_vif_info(self._session, [ 550.945732] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] for vif in network_info: [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self._sync_wrapper(fn, *args, **kwargs) [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.wait() [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self[:] = self._gt.wait() [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self._exit_event.wait() [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] result = hub.switch() [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 550.946138] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return self.greenlet.switch() [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] result = function(*args, **kwargs) [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] return func(*args, **kwargs) [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise e [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] nwinfo = self.network_api.allocate_for_instance( [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] created_port_ids = self._update_ports_for_instance( [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] with excutils.save_and_reraise_exception(): [ 550.946505] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] self.force_reraise() [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise self.value [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] updated_port = self._update_port( [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] _ensure_no_port_binding_failure(port) [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] raise exception.PortBindingFailed(port_id=port['id']) [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] nova.exception.PortBindingFailed: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. [ 550.946863] env[62736]: ERROR nova.compute.manager [instance: d995b3ce-c5b5-490e-b774-706a74632f27] [ 550.947198] env[62736]: DEBUG nova.compute.utils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 550.947536] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.849s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 550.949330] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg f07cbaf780a7442caacc18d8d781af0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.951946] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Build of instance d995b3ce-c5b5-490e-b774-706a74632f27 was re-scheduled: Binding failed for port fdbabc59-acd2-490d-95ce-d13232dc4eb2, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 550.952411] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 550.952634] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 550.952792] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 550.952932] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 550.953297] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 49727ac7dea74d07b62db998ad219d57 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 550.959797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49727ac7dea74d07b62db998ad219d57 [ 550.987595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f07cbaf780a7442caacc18d8d781af0a [ 551.108797] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 551.110531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg bec63305d83e4e7898ba06d89d766aa5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 551.137921] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 551.149865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bec63305d83e4e7898ba06d89d766aa5 [ 551.174461] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 551.174694] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 551.174841] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 551.175016] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 551.175153] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 551.175291] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 551.175645] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 551.175645] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 551.175784] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 551.175933] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 551.176119] env[62736]: DEBUG nova.virt.hardware [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 551.176984] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f3b6d7b-1282-4a71-9ab1-87f7ebe45ca5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.185271] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-309081db-9bfa-4fd1-a897-b8d95d6d936b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.383297] env[62736]: ERROR nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 551.383297] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.383297] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 551.383297] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 551.383297] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.383297] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.383297] env[62736]: ERROR nova.compute.manager raise self.value [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 551.383297] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 551.383297] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.383297] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 551.383847] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.383847] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 551.383847] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 551.383847] env[62736]: ERROR nova.compute.manager [ 551.383847] env[62736]: Traceback (most recent call last): [ 551.383847] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 551.383847] env[62736]: listener.cb(fileno) [ 551.383847] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 551.383847] env[62736]: result = function(*args, **kwargs) [ 551.383847] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 551.383847] env[62736]: return func(*args, **kwargs) [ 551.383847] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 551.383847] env[62736]: raise e [ 551.383847] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.383847] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 551.383847] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 551.383847] env[62736]: created_port_ids = self._update_ports_for_instance( [ 551.383847] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 551.383847] env[62736]: with excutils.save_and_reraise_exception(): [ 551.383847] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.383847] env[62736]: self.force_reraise() [ 551.383847] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.383847] env[62736]: raise self.value [ 551.383847] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 551.383847] env[62736]: updated_port = self._update_port( [ 551.383847] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.383847] env[62736]: _ensure_no_port_binding_failure(port) [ 551.383847] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.383847] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 551.384725] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 551.384725] env[62736]: Removing descriptor: 17 [ 551.384725] env[62736]: ERROR nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Traceback (most recent call last): [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] yield resources [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.driver.spawn(context, instance, image_meta, [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self._vmops.spawn(context, instance, image_meta, injected_files, [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 551.384725] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] vm_ref = self.build_virtual_machine(instance, [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] vif_infos = vmwarevif.get_vif_info(self._session, [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] for vif in network_info: [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self._sync_wrapper(fn, *args, **kwargs) [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.wait() [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self[:] = self._gt.wait() [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self._exit_event.wait() [ 551.385091] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] result = hub.switch() [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self.greenlet.switch() [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] result = function(*args, **kwargs) [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return func(*args, **kwargs) [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise e [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] nwinfo = self.network_api.allocate_for_instance( [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 551.385436] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] created_port_ids = self._update_ports_for_instance( [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] with excutils.save_and_reraise_exception(): [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.force_reraise() [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise self.value [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] updated_port = self._update_port( [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] _ensure_no_port_binding_failure(port) [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 551.385776] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise exception.PortBindingFailed(port_id=port['id']) [ 551.386268] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 551.386268] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] [ 551.386268] env[62736]: INFO nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Terminating instance [ 551.386546] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 551.386699] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 551.386866] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 551.387304] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 4c14b6c475104a758f442927902d2dad in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 551.394288] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c14b6c475104a758f442927902d2dad [ 551.532510] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 551.560875] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Successfully created port: 42afc0e2-7da7-431c-a861-5c77b0a53036 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 551.633957] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.766563] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78d39a36-934c-4cb3-a008-06139b51b399 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.777576] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a78a132d-0823-4a41-8807-c8425dafb4db {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.806722] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec5af260-1686-4012-be90-60a61980335f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.814314] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8117dcd-897e-4fa5-9fb8-c9426d4394eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 551.829039] env[62736]: DEBUG nova.compute.provider_tree [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 551.829608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d6ca0e78ab6b489e8ee47c65def1dfc9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 551.839776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6ca0e78ab6b489e8ee47c65def1dfc9 [ 551.855236] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 551.855716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 81a50aafa6154687add3e2dfd47e1cd4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 551.866850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81a50aafa6154687add3e2dfd47e1cd4 [ 551.934746] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 552.209168] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 552.209871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 21e1e986055b4c9dbcb328ac9f418112 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.218901] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21e1e986055b4c9dbcb328ac9f418112 [ 552.337686] env[62736]: DEBUG nova.scheduler.client.report [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 552.340187] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 3c75d261a0b04f72a84d4799d5f4dbe4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.349973] env[62736]: DEBUG nova.compute.manager [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Received event network-changed-3051a2c4-e501-45c9-b625-6fcae110ce27 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 552.350169] env[62736]: DEBUG nova.compute.manager [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Refreshing instance network info cache due to event network-changed-3051a2c4-e501-45c9-b625-6fcae110ce27. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 552.350485] env[62736]: DEBUG oslo_concurrency.lockutils [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] Acquiring lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 552.350641] env[62736]: DEBUG oslo_concurrency.lockutils [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] Acquired lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 552.350780] env[62736]: DEBUG nova.network.neutron [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Refreshing network info cache for port 3051a2c4-e501-45c9-b625-6fcae110ce27 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 552.351257] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] Expecting reply to msg 1008c95c2bb04ce0b8b9eaf1edbb483a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.353037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c75d261a0b04f72a84d4799d5f4dbe4 [ 552.357285] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-d995b3ce-c5b5-490e-b774-706a74632f27" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 552.358099] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 552.358099] env[62736]: DEBUG nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 552.358099] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 552.360130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1008c95c2bb04ce0b8b9eaf1edbb483a [ 552.395980] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 552.396577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg a2b4ac9eeddc4ca097ff044f84c448ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.403589] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2b4ac9eeddc4ca097ff044f84c448ca [ 552.712083] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 552.712520] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 552.712701] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 552.713191] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b77fef16-3a8e-4c31-9d8d-f072e7db6189 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.723234] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-452f6c6a-44ea-4c53-a4b8-096a39431448 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.747691] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ae1041c6-888a-4cb3-8da3-00e43b5b6531 could not be found. [ 552.747913] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 552.748123] env[62736]: INFO nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Took 0.04 seconds to destroy the instance on the hypervisor. [ 552.748364] env[62736]: DEBUG oslo.service.loopingcall [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 552.748586] env[62736]: DEBUG nova.compute.manager [-] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 552.748673] env[62736]: DEBUG nova.network.neutron [-] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 552.781886] env[62736]: DEBUG nova.network.neutron [-] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 552.782448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 02b07c86e902459bbf33482bac180b2c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.790263] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02b07c86e902459bbf33482bac180b2c [ 552.853727] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.906s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 552.854427] env[62736]: ERROR nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Traceback (most recent call last): [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.driver.spawn(context, instance, image_meta, [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] vm_ref = self.build_virtual_machine(instance, [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] vif_infos = vmwarevif.get_vif_info(self._session, [ 552.854427] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] for vif in network_info: [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self._sync_wrapper(fn, *args, **kwargs) [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.wait() [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self[:] = self._gt.wait() [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self._exit_event.wait() [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] result = hub.switch() [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 552.854735] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return self.greenlet.switch() [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] result = function(*args, **kwargs) [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] return func(*args, **kwargs) [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise e [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] nwinfo = self.network_api.allocate_for_instance( [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] created_port_ids = self._update_ports_for_instance( [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] with excutils.save_and_reraise_exception(): [ 552.855064] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] self.force_reraise() [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise self.value [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] updated_port = self._update_port( [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] _ensure_no_port_binding_failure(port) [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] raise exception.PortBindingFailed(port_id=port['id']) [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] nova.exception.PortBindingFailed: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. [ 552.855385] env[62736]: ERROR nova.compute.manager [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] [ 552.855715] env[62736]: DEBUG nova.compute.utils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 552.858207] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 16.365s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.858439] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 552.858605] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 552.858879] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.191s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 552.861257] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 78c3b8c4243a4c29a7ac698601601aa7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.864123] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Build of instance 01127e18-3d68-4b50-b0d4-89f186aa7fb4 was re-scheduled: Binding failed for port a989424f-fd97-4857-9982-210001ac2adc, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 552.864123] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 552.864123] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 552.864123] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 552.864123] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 552.864416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg cd6fd286e4f848fa8d7dd8507f6ef52f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.873358] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f343b189-b812-4004-8cc3-ae72edb64d67 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.877072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd6fd286e4f848fa8d7dd8507f6ef52f [ 552.887319] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f79df1b-408b-40ff-adc4-b6d04f43b3b1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.901698] env[62736]: DEBUG nova.network.neutron [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 552.902217] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 0047658eef094462ac5e7aff54d80985 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 552.904654] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 552.906995] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e300fa4-edb6-42b0-9e7b-24f587ad756a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.914005] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba3ee6db-3a41-4851-bfab-ae8e55725536 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.919281] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0047658eef094462ac5e7aff54d80985 [ 552.956643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78c3b8c4243a4c29a7ac698601601aa7 [ 552.957244] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181772MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 552.957387] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 552.959081] env[62736]: DEBUG nova.network.neutron [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 552.961945] env[62736]: DEBUG nova.compute.manager [None req-f9be2840-f7fb-4026-ba5e-c2af74072756 tempest-ServerDiagnosticsV248Test-134475782 tempest-ServerDiagnosticsV248Test-134475782-project-admin] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 552.962944] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ed6e0b5-02ab-4404-9ff6-6bc167169785 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 552.970880] env[62736]: INFO nova.compute.manager [None req-f9be2840-f7fb-4026-ba5e-c2af74072756 tempest-ServerDiagnosticsV248Test-134475782 tempest-ServerDiagnosticsV248Test-134475782-project-admin] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Retrieving diagnostics [ 552.971626] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6968b495-82fb-41bb-b105-54c306b03c2f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.003357] env[62736]: ERROR nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 553.003357] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.003357] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.003357] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.003357] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.003357] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.003357] env[62736]: ERROR nova.compute.manager raise self.value [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.003357] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 553.003357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.003357] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 553.003861] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.003861] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 553.003861] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 553.003861] env[62736]: ERROR nova.compute.manager [ 553.003861] env[62736]: Traceback (most recent call last): [ 553.003861] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 553.003861] env[62736]: listener.cb(fileno) [ 553.003861] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 553.003861] env[62736]: result = function(*args, **kwargs) [ 553.003861] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 553.003861] env[62736]: return func(*args, **kwargs) [ 553.003861] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 553.003861] env[62736]: raise e [ 553.003861] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.003861] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 553.003861] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.003861] env[62736]: created_port_ids = self._update_ports_for_instance( [ 553.003861] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.003861] env[62736]: with excutils.save_and_reraise_exception(): [ 553.003861] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.003861] env[62736]: self.force_reraise() [ 553.003861] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.003861] env[62736]: raise self.value [ 553.003861] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.003861] env[62736]: updated_port = self._update_port( [ 553.003861] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.003861] env[62736]: _ensure_no_port_binding_failure(port) [ 553.003861] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.003861] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 553.004846] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 553.004846] env[62736]: Removing descriptor: 15 [ 553.004846] env[62736]: ERROR nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Traceback (most recent call last): [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] yield resources [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.driver.spawn(context, instance, image_meta, [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 553.004846] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] vm_ref = self.build_virtual_machine(instance, [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] vif_infos = vmwarevif.get_vif_info(self._session, [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] for vif in network_info: [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self._sync_wrapper(fn, *args, **kwargs) [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.wait() [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self[:] = self._gt.wait() [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self._exit_event.wait() [ 553.005237] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] result = hub.switch() [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self.greenlet.switch() [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] result = function(*args, **kwargs) [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return func(*args, **kwargs) [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise e [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] nwinfo = self.network_api.allocate_for_instance( [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.005621] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] created_port_ids = self._update_ports_for_instance( [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] with excutils.save_and_reraise_exception(): [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.force_reraise() [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise self.value [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] updated_port = self._update_port( [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] _ensure_no_port_binding_failure(port) [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.005995] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise exception.PortBindingFailed(port_id=port['id']) [ 553.006337] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 553.006337] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] [ 553.006337] env[62736]: INFO nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Terminating instance [ 553.006337] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 553.006469] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 553.006588] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 553.006979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 52d4107ff0dd449abc0320995538ae26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.013900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52d4107ff0dd449abc0320995538ae26 [ 553.223469] env[62736]: DEBUG nova.network.neutron [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.224074] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] Expecting reply to msg 7425ea9e0ecc4da0ae58679322bcc4a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.233496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7425ea9e0ecc4da0ae58679322bcc4a1 [ 553.284496] env[62736]: DEBUG nova.network.neutron [-] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.284977] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f0b40735cf884368a906dd6af1bffa17 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.293078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0b40735cf884368a906dd6af1bffa17 [ 553.307789] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.308347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 2d7236c1c5a44f44ade1fe447a3c6e54 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.316983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d7236c1c5a44f44ade1fe447a3c6e54 [ 553.405295] env[62736]: INFO nova.compute.manager [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: d995b3ce-c5b5-490e-b774-706a74632f27] Took 1.05 seconds to deallocate network for instance. [ 553.407054] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 74c8b1ab0b9d4d068dc221848f2b253b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.448783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74c8b1ab0b9d4d068dc221848f2b253b [ 553.535959] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 553.640154] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4166f7ae-f751-45d4-8734-de1aafa8344b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.643615] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 553.644658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 97cfba49e75f48418361b3b5bf253e69 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.652150] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f52f3fda-7884-4d08-a98b-32adda6e60d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.656950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97cfba49e75f48418361b3b5bf253e69 [ 553.688368] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30afd3a8-369b-4e56-aad1-bf6789b1eba4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.695546] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c46a91af-2980-4970-80c7-1b5aecbf6f2b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 553.709830] env[62736]: DEBUG nova.compute.provider_tree [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 553.710612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg ba35f7b14fde46bebc1260c0fac395e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.717961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba35f7b14fde46bebc1260c0fac395e9 [ 553.726289] env[62736]: DEBUG oslo_concurrency.lockutils [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] Releasing lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 553.726515] env[62736]: DEBUG nova.compute.manager [req-59455a06-a11b-46a7-b2f1-954307254412 req-964b158e-de56-4f8e-a12d-43040c578d44 service nova] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Received event network-vif-deleted-3051a2c4-e501-45c9-b625-6fcae110ce27 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 553.787627] env[62736]: INFO nova.compute.manager [-] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Took 1.04 seconds to deallocate network for instance. [ 553.790177] env[62736]: DEBUG nova.compute.claims [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 553.790537] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 553.810294] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-01127e18-3d68-4b50-b0d4-89f186aa7fb4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 553.810518] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 553.810704] env[62736]: DEBUG nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 553.810868] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 553.911488] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 107b70b0f61b494e817facb872f55a05 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.966639] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 107b70b0f61b494e817facb872f55a05 [ 553.979309] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 553.979925] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg c2e47805854e40cd99d84e9ec7705b00 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.981681] env[62736]: ERROR nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 553.981681] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.981681] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.981681] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.981681] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.981681] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.981681] env[62736]: ERROR nova.compute.manager raise self.value [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.981681] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 553.981681] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.981681] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 553.982167] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.982167] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 553.982167] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 553.982167] env[62736]: ERROR nova.compute.manager [ 553.982167] env[62736]: Traceback (most recent call last): [ 553.982167] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 553.982167] env[62736]: listener.cb(fileno) [ 553.982167] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 553.982167] env[62736]: result = function(*args, **kwargs) [ 553.982167] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 553.982167] env[62736]: return func(*args, **kwargs) [ 553.982167] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 553.982167] env[62736]: raise e [ 553.982167] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.982167] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 553.982167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.982167] env[62736]: created_port_ids = self._update_ports_for_instance( [ 553.982167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.982167] env[62736]: with excutils.save_and_reraise_exception(): [ 553.982167] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.982167] env[62736]: self.force_reraise() [ 553.982167] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.982167] env[62736]: raise self.value [ 553.982167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.982167] env[62736]: updated_port = self._update_port( [ 553.982167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.982167] env[62736]: _ensure_no_port_binding_failure(port) [ 553.982167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.982167] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 553.982930] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 553.982930] env[62736]: Removing descriptor: 19 [ 553.982930] env[62736]: ERROR nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Traceback (most recent call last): [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] yield resources [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.driver.spawn(context, instance, image_meta, [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 553.982930] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] vm_ref = self.build_virtual_machine(instance, [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] vif_infos = vmwarevif.get_vif_info(self._session, [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] for vif in network_info: [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self._sync_wrapper(fn, *args, **kwargs) [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.wait() [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self[:] = self._gt.wait() [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self._exit_event.wait() [ 553.983259] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] result = hub.switch() [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self.greenlet.switch() [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] result = function(*args, **kwargs) [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return func(*args, **kwargs) [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise e [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] nwinfo = self.network_api.allocate_for_instance( [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 553.983603] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] created_port_ids = self._update_ports_for_instance( [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] with excutils.save_and_reraise_exception(): [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.force_reraise() [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise self.value [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] updated_port = self._update_port( [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] _ensure_no_port_binding_failure(port) [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 553.984067] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise exception.PortBindingFailed(port_id=port['id']) [ 553.984421] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 553.984421] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] [ 553.984421] env[62736]: INFO nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Terminating instance [ 553.984912] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 553.985068] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquired lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 553.985226] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 553.985608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 8a009dcfd9fd40d2a444943e295108c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 553.988509] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2e47805854e40cd99d84e9ec7705b00 [ 553.992685] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a009dcfd9fd40d2a444943e295108c3 [ 554.152015] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 554.152465] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 554.152685] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 554.152944] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-21ced4d7-8d06-46a5-99bf-6c1145fe8d54 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.162336] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d5726e4-bdea-4c27-872f-58a58685e48a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 554.191399] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance aa9671f3-07a8-422f-93b5-006de0f2cb3c could not be found. [ 554.191586] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 554.191763] env[62736]: INFO nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 554.192079] env[62736]: DEBUG oslo.service.loopingcall [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 554.192329] env[62736]: DEBUG nova.compute.manager [-] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 554.192424] env[62736]: DEBUG nova.network.neutron [-] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 554.206631] env[62736]: DEBUG nova.network.neutron [-] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 554.207145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 006e8496c26147c6ab4ea58c37553d13 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.213329] env[62736]: DEBUG nova.scheduler.client.report [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 554.215746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg b35a15c119ad49d8a2535f5c324b2b0d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.216799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 006e8496c26147c6ab4ea58c37553d13 [ 554.227335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b35a15c119ad49d8a2535f5c324b2b0d [ 554.438734] env[62736]: INFO nova.scheduler.client.report [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Deleted allocations for instance d995b3ce-c5b5-490e-b774-706a74632f27 [ 554.444827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg abdfc14f0f244955aed6de1cd19eba9b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.475140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abdfc14f0f244955aed6de1cd19eba9b [ 554.482678] env[62736]: DEBUG nova.network.neutron [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.483197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg cbeffd8af4cd4b14a8a151d476793ea9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.501759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbeffd8af4cd4b14a8a151d476793ea9 [ 554.537910] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 554.597779] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.598314] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg ae19032017844eac904dafad8f76bdc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.607441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae19032017844eac904dafad8f76bdc6 [ 554.708982] env[62736]: DEBUG nova.network.neutron [-] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.709500] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 521e0384a0dc41d4baffe580f8aa3470 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.718146] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 521e0384a0dc41d4baffe580f8aa3470 [ 554.719029] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.860s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.719655] env[62736]: ERROR nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Traceback (most recent call last): [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.driver.spawn(context, instance, image_meta, [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] vm_ref = self.build_virtual_machine(instance, [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] vif_infos = vmwarevif.get_vif_info(self._session, [ 554.719655] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] for vif in network_info: [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self._sync_wrapper(fn, *args, **kwargs) [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.wait() [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self[:] = self._gt.wait() [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self._exit_event.wait() [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] result = hub.switch() [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 554.719979] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return self.greenlet.switch() [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] result = function(*args, **kwargs) [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] return func(*args, **kwargs) [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise e [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] nwinfo = self.network_api.allocate_for_instance( [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] created_port_ids = self._update_ports_for_instance( [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] with excutils.save_and_reraise_exception(): [ 554.720336] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] self.force_reraise() [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise self.value [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] updated_port = self._update_port( [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] _ensure_no_port_binding_failure(port) [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] raise exception.PortBindingFailed(port_id=port['id']) [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] nova.exception.PortBindingFailed: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. [ 554.720684] env[62736]: ERROR nova.compute.manager [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] [ 554.720974] env[62736]: DEBUG nova.compute.utils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 554.721502] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.119s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.723267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 442dc3bd9e4e4f0aa763e5a0897918e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.731154] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Build of instance 03b95ad5-85fe-41ad-987f-8140380f0a0d was re-scheduled: Binding failed for port b10f2a3c-b187-4aa0-9d8d-5fbcf18d8c1f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 554.731154] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 554.731154] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquiring lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 554.731154] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Acquired lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 554.731154] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 554.731553] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg cb890c5c0e2b4226b8edecc627b68004 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.734946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb890c5c0e2b4226b8edecc627b68004 [ 554.782153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 442dc3bd9e4e4f0aa763e5a0897918e6 [ 554.950054] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cb5fb93-4f7d-4a5a-970c-db1b2fe4740e tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "d995b3ce-c5b5-490e-b774-706a74632f27" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 29.449s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.950054] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 360bdf6f46bf4aa082a9af6d61b10eaa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 554.968576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 360bdf6f46bf4aa082a9af6d61b10eaa [ 554.985038] env[62736]: INFO nova.compute.manager [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 01127e18-3d68-4b50-b0d4-89f186aa7fb4] Took 1.17 seconds to deallocate network for instance. [ 554.986768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg aeaf1f305e8e445599d8c226f5005a65 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.020698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aeaf1f305e8e445599d8c226f5005a65 [ 555.100995] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Releasing lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 555.101632] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 555.101930] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 555.102247] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4c4d2532-f5e7-404c-95b1-5685437479a1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.115968] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1936c553-2dec-4d79-a600-46f3fc60e1f5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.149868] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a5a690d3-8018-4cc5-8d59-55062a595ed0 could not be found. [ 555.150215] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 555.150479] env[62736]: INFO nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Took 0.05 seconds to destroy the instance on the hypervisor. [ 555.150761] env[62736]: DEBUG oslo.service.loopingcall [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 555.150993] env[62736]: DEBUG nova.compute.manager [-] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 555.151089] env[62736]: DEBUG nova.network.neutron [-] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 555.177799] env[62736]: DEBUG nova.network.neutron [-] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 555.178458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d664329dc985480f9c03fb9579a77af9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.185089] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d664329dc985480f9c03fb9579a77af9 [ 555.224545] env[62736]: INFO nova.compute.manager [-] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Took 1.03 seconds to deallocate network for instance. [ 555.229679] env[62736]: DEBUG nova.compute.claims [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 555.229956] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.381659] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquiring lock "3a848010-382b-4753-8420-37222963e7e2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.382306] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "3a848010-382b-4753-8420-37222963e7e2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.400719] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 555.454116] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 555.454116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg b8169a45335b4b62a3ba0f5d8865c18d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.479576] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Received event network-changed-374282b5-a930-4685-a3e9-c381d14b89c8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 555.479576] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Refreshing instance network info cache due to event network-changed-374282b5-a930-4685-a3e9-c381d14b89c8. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 555.479576] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquiring lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 555.480948] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquired lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 555.480948] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Refreshing network info cache for port 374282b5-a930-4685-a3e9-c381d14b89c8 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 555.480948] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg 5d320b8a60404823b4ce2e1553396259 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.494611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 74837789719e4885b5dcb7220cb6cd1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.498621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d320b8a60404823b4ce2e1553396259 [ 555.548575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74837789719e4885b5dcb7220cb6cd1d [ 555.557599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8169a45335b4b62a3ba0f5d8865c18d [ 555.577505] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cae43bac-0852-41fd-9675-18caa2678f51 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.588881] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e0f187c-0a2b-4072-808e-8f1e99360cb9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.634253] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c02de4bf-d519-48ba-ae1d-e180aecfa1ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.642824] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb1cffcd-ea8e-4aff-8171-194f8d395318 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 555.657249] env[62736]: DEBUG nova.compute.provider_tree [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 555.658060] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 36a05539a94c4d78968ee9f45646465c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.666031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36a05539a94c4d78968ee9f45646465c [ 555.680661] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 555.681370] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg f86937f9215142c0ad888fdd7281905f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.682343] env[62736]: DEBUG nova.network.neutron [-] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 555.687312] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c1a4ff164f524c3fab872621b087636b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.695840] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f86937f9215142c0ad888fdd7281905f [ 555.695840] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Releasing lock "refresh_cache-03b95ad5-85fe-41ad-987f-8140380f0a0d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 555.695840] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 555.695840] env[62736]: DEBUG nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 555.695840] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 555.703187] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1a4ff164f524c3fab872621b087636b [ 555.719043] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 555.719659] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 2444e5d9b8a64ea5b10bc52e8ee583b1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 555.726957] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2444e5d9b8a64ea5b10bc52e8ee583b1 [ 555.972791] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.030589] env[62736]: INFO nova.scheduler.client.report [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Deleted allocations for instance 01127e18-3d68-4b50-b0d4-89f186aa7fb4 [ 556.031394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 4221ac39685a4bdbb51568d66561c983 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.047522] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4221ac39685a4bdbb51568d66561c983 [ 556.049461] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 556.164884] env[62736]: DEBUG nova.scheduler.client.report [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 556.164884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 4d1ba7b71a4d4d1583a5bef4dbd1fdf1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.179006] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d1ba7b71a4d4d1583a5bef4dbd1fdf1 [ 556.196237] env[62736]: INFO nova.compute.manager [-] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Took 1.04 seconds to deallocate network for instance. [ 556.199029] env[62736]: DEBUG nova.compute.claims [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 556.199347] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.222934] env[62736]: DEBUG nova.network.neutron [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 556.223592] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 4439131c62fd496082c1d473eb546b37 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.234381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4439131c62fd496082c1d473eb546b37 [ 556.328591] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 556.329059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg 99bcd34e1fe847e29313e88b2ae47ecf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.330145] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquiring lock "c7670280-b5a7-4de2-8cf9-6814a81eb1a6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.330532] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "c7670280-b5a7-4de2-8cf9-6814a81eb1a6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.337280] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99bcd34e1fe847e29313e88b2ae47ecf [ 556.533867] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a3b85042-ae47-49c2-846d-542ff6197062 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "01127e18-3d68-4b50-b0d4-89f186aa7fb4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.871s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.534486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 0b957c8a0bc74e85ab7329e20883c22d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.546448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b957c8a0bc74e85ab7329e20883c22d [ 556.667030] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.945s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.667791] env[62736]: ERROR nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Traceback (most recent call last): [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.driver.spawn(context, instance, image_meta, [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] vm_ref = self.build_virtual_machine(instance, [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 556.667791] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] for vif in network_info: [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self._sync_wrapper(fn, *args, **kwargs) [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.wait() [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self[:] = self._gt.wait() [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self._exit_event.wait() [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] result = hub.switch() [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 556.668160] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return self.greenlet.switch() [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] result = function(*args, **kwargs) [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] return func(*args, **kwargs) [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise e [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] nwinfo = self.network_api.allocate_for_instance( [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] created_port_ids = self._update_ports_for_instance( [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] with excutils.save_and_reraise_exception(): [ 556.668506] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] self.force_reraise() [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise self.value [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] updated_port = self._update_port( [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] _ensure_no_port_binding_failure(port) [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] raise exception.PortBindingFailed(port_id=port['id']) [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] nova.exception.PortBindingFailed: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. [ 556.668931] env[62736]: ERROR nova.compute.manager [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] [ 556.669213] env[62736]: DEBUG nova.compute.utils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 556.670112] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.611s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.671834] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 7f43c70b434c4940a85bdbea2773ce2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.673208] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Build of instance 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2 was re-scheduled: Binding failed for port f558dc42-a489-4138-8f8c-cfef1c9c730b, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 556.673734] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 556.674040] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquiring lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 556.674107] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Acquired lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 556.674332] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 556.674737] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg ceb83065b540465f801e66041002e86d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.686927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ceb83065b540465f801e66041002e86d [ 556.716764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f43c70b434c4940a85bdbea2773ce2b [ 556.735493] env[62736]: INFO nova.compute.manager [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] [instance: 03b95ad5-85fe-41ad-987f-8140380f0a0d] Took 1.04 seconds to deallocate network for instance. [ 556.737189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 9ba1bbb757af4bf68654e0b97d07bdd7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.770041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ba1bbb757af4bf68654e0b97d07bdd7 [ 556.831486] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Releasing lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 556.833629] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Received event network-vif-deleted-374282b5-a930-4685-a3e9-c381d14b89c8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 556.833629] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Received event network-changed-ea932e45-f256-494c-9fc5-5ace73092f2f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 556.833629] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Refreshing instance network info cache due to event network-changed-ea932e45-f256-494c-9fc5-5ace73092f2f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 556.833629] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquiring lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 556.833629] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquired lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 556.833877] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Refreshing network info cache for port ea932e45-f256-494c-9fc5-5ace73092f2f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 556.833877] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg bc1087687d15469badbf87c09c6a1c1c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 556.840172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc1087687d15469badbf87c09c6a1c1c [ 557.040683] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 557.042433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 1aad8cc4501742b780f0568d05051722 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.084665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aad8cc4501742b780f0568d05051722 [ 557.207459] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 557.242260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 81e0fe6fb0ec4147bb2b742e0a2a1f20 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.278544] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 557.279124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg e6895350e54f4dfb94bf6094db80921c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.280525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81e0fe6fb0ec4147bb2b742e0a2a1f20 [ 557.290593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6895350e54f4dfb94bf6094db80921c [ 557.348942] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 557.425145] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3279bd3-a6bc-4cda-b806-14cf14678cb4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.432560] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc0da281-2b14-488b-bb9d-0a65998b5fba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.436827] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 557.437296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg 0ce394005493473b8b60424aee7fd995 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.467056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ce394005493473b8b60424aee7fd995 [ 557.468041] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99bf7eb8-4095-4f06-b9f2-828d8fe3268f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.475621] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aca43a8b-5522-4655-bea5-0ba1946ea3a3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 557.489350] env[62736]: DEBUG nova.compute.provider_tree [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 557.489871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 4ce7aacc80f24087b4e034638a0c6d70 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.498045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ce7aacc80f24087b4e034638a0c6d70 [ 557.561453] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.774261] env[62736]: INFO nova.scheduler.client.report [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Deleted allocations for instance 03b95ad5-85fe-41ad-987f-8140380f0a0d [ 557.780707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Expecting reply to msg 1f926b3b6e1b4ad19c83591218c1ab52 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.781633] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Releasing lock "refresh_cache-1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 557.781840] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 557.781997] env[62736]: DEBUG nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 557.782225] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 557.797890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f926b3b6e1b4ad19c83591218c1ab52 [ 557.806060] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 557.806684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 38ec7ab4e5f34d70862d455ea7f8f2e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.813334] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38ec7ab4e5f34d70862d455ea7f8f2e1 [ 557.939315] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Releasing lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 557.939609] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Received event network-vif-deleted-ea932e45-f256-494c-9fc5-5ace73092f2f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 557.939792] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Received event network-changed-42afc0e2-7da7-431c-a861-5c77b0a53036 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 557.939951] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Refreshing instance network info cache due to event network-changed-42afc0e2-7da7-431c-a861-5c77b0a53036. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 557.940179] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquiring lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 557.940316] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Acquired lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 557.940495] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Refreshing network info cache for port 42afc0e2-7da7-431c-a861-5c77b0a53036 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 557.940931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg 47d71e7f7b2b468c9dc5dbaae6418ace in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 557.948104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47d71e7f7b2b468c9dc5dbaae6418ace [ 557.992413] env[62736]: DEBUG nova.scheduler.client.report [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 557.994958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 3a69c266930343eaa78e32e69d4bb220 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.013531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a69c266930343eaa78e32e69d4bb220 [ 558.285717] env[62736]: DEBUG oslo_concurrency.lockutils [None req-49aff994-c961-441e-80b7-68437e3a1ee8 tempest-ServerTagsTestJSON-682251749 tempest-ServerTagsTestJSON-682251749-project-member] Lock "03b95ad5-85fe-41ad-987f-8140380f0a0d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.908s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 558.285960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 21621d6ee8a74220a9998a2cbbdfe908 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.297329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21621d6ee8a74220a9998a2cbbdfe908 [ 558.312572] env[62736]: DEBUG nova.network.neutron [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 558.312572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 54230d028a57433fa716b67e09e42f08 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.318879] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54230d028a57433fa716b67e09e42f08 [ 558.461647] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 558.498856] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.829s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 558.499573] env[62736]: ERROR nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Traceback (most recent call last): [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.driver.spawn(context, instance, image_meta, [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self._vmops.spawn(context, instance, image_meta, injected_files, [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] vm_ref = self.build_virtual_machine(instance, [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] vif_infos = vmwarevif.get_vif_info(self._session, [ 558.499573] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] for vif in network_info: [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self._sync_wrapper(fn, *args, **kwargs) [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.wait() [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self[:] = self._gt.wait() [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self._exit_event.wait() [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] result = hub.switch() [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 558.499885] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return self.greenlet.switch() [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] result = function(*args, **kwargs) [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] return func(*args, **kwargs) [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise e [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] nwinfo = self.network_api.allocate_for_instance( [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] created_port_ids = self._update_ports_for_instance( [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] with excutils.save_and_reraise_exception(): [ 558.500231] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] self.force_reraise() [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise self.value [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] updated_port = self._update_port( [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] _ensure_no_port_binding_failure(port) [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] raise exception.PortBindingFailed(port_id=port['id']) [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] nova.exception.PortBindingFailed: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. [ 558.500594] env[62736]: ERROR nova.compute.manager [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] [ 558.500929] env[62736]: DEBUG nova.compute.utils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 558.501681] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.764s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 558.503929] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg b06637e727f44597a0a7896f785a0e41 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.504843] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Build of instance 2eb46a2a-27fc-4d44-9d18-6d4553026a60 was re-scheduled: Binding failed for port d7e16639-ee52-48ff-bbd6-ffe2b267f1d7, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 558.505402] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 558.505544] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 558.505645] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquired lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 558.505800] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 558.506352] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 91709de1e4ce4a248af8ae0e5e41fbef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.513445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91709de1e4ce4a248af8ae0e5e41fbef [ 558.538513] env[62736]: DEBUG nova.network.neutron [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 558.539110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Expecting reply to msg 7aa46feb639746b9bd46043c2b65d903 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.548285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b06637e727f44597a0a7896f785a0e41 [ 558.551977] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7aa46feb639746b9bd46043c2b65d903 [ 558.788589] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 558.790318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 4d6753b591ab4e87a02ae214d82b0044 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.812844] env[62736]: INFO nova.compute.manager [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] [instance: 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2] Took 1.03 seconds to deallocate network for instance. [ 558.814497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg b09431f18d64413c85bdfe14e638ae29 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 558.841204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d6753b591ab4e87a02ae214d82b0044 [ 558.849845] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b09431f18d64413c85bdfe14e638ae29 [ 559.042240] env[62736]: DEBUG oslo_concurrency.lockutils [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] Releasing lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 559.042490] env[62736]: DEBUG nova.compute.manager [req-83509b76-6876-453e-9152-a6b7246f17a3 req-240eb0f3-5daf-47bd-a9fc-2c4196dad018 service nova] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Received event network-vif-deleted-42afc0e2-7da7-431c-a861-5c77b0a53036 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 559.059500] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 559.146725] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 559.146933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 7bd38f1c45134356aff9ddcf6e9fe7a2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.156749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bd38f1c45134356aff9ddcf6e9fe7a2 [ 559.250209] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5248d6d1-bf2f-4246-b08b-6f53bbcc1ed6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.259057] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14dc9881-4984-4a3c-a549-f0d58a047fd6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.295743] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e6574b6-ad88-463e-b735-218fc68df371 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.307474] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74b076de-85f2-43be-ab66-aa89ac50b9b6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 559.313231] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 559.326986] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg ef74d6c1c6fc41f5bbb9b7891c0885ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.337147] env[62736]: DEBUG nova.compute.provider_tree [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 559.337777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 987142b3a7ad42fda89470802c6e055d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.347702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 987142b3a7ad42fda89470802c6e055d [ 559.366243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef74d6c1c6fc41f5bbb9b7891c0885ee [ 559.650745] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Releasing lock "refresh_cache-2eb46a2a-27fc-4d44-9d18-6d4553026a60" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 559.650996] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 559.651164] env[62736]: DEBUG nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 559.651340] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 559.679699] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 559.680367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 719f6ec8888f4219ac06428a717113d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.693306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 719f6ec8888f4219ac06428a717113d2 [ 559.843177] env[62736]: DEBUG nova.scheduler.client.report [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 559.845586] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 0cc2d4d40977451bbbc2f9bb16dd3938 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.862831] env[62736]: INFO nova.scheduler.client.report [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Deleted allocations for instance 1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2 [ 559.868487] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Expecting reply to msg 1e6860f6054b4032ba1aa5a47c1a56b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 559.869488] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cc2d4d40977451bbbc2f9bb16dd3938 [ 559.890475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e6860f6054b4032ba1aa5a47c1a56b3 [ 559.957475] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "558058e0-9808-415a-8866-75d1c8598d43" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 559.957475] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "558058e0-9808-415a-8866-75d1c8598d43" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.184747] env[62736]: DEBUG nova.network.neutron [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 560.184747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 17de5fe2c8124815a2c2a9804d2265d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.193140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17de5fe2c8124815a2c2a9804d2265d5 [ 560.348646] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.847s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.349373] env[62736]: ERROR nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] Traceback (most recent call last): [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.driver.spawn(context, instance, image_meta, [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self._vmops.spawn(context, instance, image_meta, injected_files, [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] vm_ref = self.build_virtual_machine(instance, [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] vif_infos = vmwarevif.get_vif_info(self._session, [ 560.349373] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] for vif in network_info: [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self._sync_wrapper(fn, *args, **kwargs) [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.wait() [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self[:] = self._gt.wait() [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self._exit_event.wait() [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] result = hub.switch() [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 560.349740] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return self.greenlet.switch() [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] result = function(*args, **kwargs) [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] return func(*args, **kwargs) [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise e [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] nwinfo = self.network_api.allocate_for_instance( [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] created_port_ids = self._update_ports_for_instance( [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] with excutils.save_and_reraise_exception(): [ 560.350064] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] self.force_reraise() [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise self.value [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] updated_port = self._update_port( [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] _ensure_no_port_binding_failure(port) [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] raise exception.PortBindingFailed(port_id=port['id']) [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] nova.exception.PortBindingFailed: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. [ 560.350388] env[62736]: ERROR nova.compute.manager [instance: 0b211be4-1955-456d-bf30-5195b5169807] [ 560.350670] env[62736]: DEBUG nova.compute.utils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 560.351317] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.718s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.352753] env[62736]: INFO nova.compute.claims [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 560.354303] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 1fefee34e1ac4971924dc0a6cb3b35be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.356011] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Build of instance 0b211be4-1955-456d-bf30-5195b5169807 was re-scheduled: Binding failed for port 3051a2c4-e501-45c9-b625-6fcae110ce27, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 560.356575] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 560.356801] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquiring lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 560.356945] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Acquired lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 560.357098] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 560.357463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 26782bed5fbb46269c09f3720f54ae1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.363632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26782bed5fbb46269c09f3720f54ae1e [ 560.374329] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7a66ceda-251c-4951-be29-f56f4cbbc493 tempest-ServerDiagnosticsNegativeTest-1606981752 tempest-ServerDiagnosticsNegativeTest-1606981752-project-member] Lock "1b8498b4-5186-4c0e-9b0b-9b8812e8a2e2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 33.666s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.374832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg f76184fd1af64de5a73ff24eebdd322b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.391912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f76184fd1af64de5a73ff24eebdd322b [ 560.397132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1fefee34e1ac4971924dc0a6cb3b35be [ 560.689836] env[62736]: INFO nova.compute.manager [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: 2eb46a2a-27fc-4d44-9d18-6d4553026a60] Took 1.03 seconds to deallocate network for instance. [ 560.689836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 327699d603ea43729e89289576812941 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.737117] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 327699d603ea43729e89289576812941 [ 560.862912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg fa64ea22e3ce4375afb0fc385827ef8a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.872220] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa64ea22e3ce4375afb0fc385827ef8a [ 560.876288] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 560.878651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg fbaa4f93b36a48a982c823be3ef560cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 560.898980] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 560.937994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbaa4f93b36a48a982c823be3ef560cf [ 561.018558] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 561.018558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 734e5320873049e894dc9df86f3878b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 561.028672] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 734e5320873049e894dc9df86f3878b8 [ 561.194032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 8396cb8cf39646b7b3d528a0c44a794b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 561.227302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8396cb8cf39646b7b3d528a0c44a794b [ 561.394885] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.520318] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Releasing lock "refresh_cache-0b211be4-1955-456d-bf30-5195b5169807" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 561.520546] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 561.520764] env[62736]: DEBUG nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 561.520961] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 561.542228] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 561.542786] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 69457eb80e0f407d8707ed88eb7bd580 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 561.549577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69457eb80e0f407d8707ed88eb7bd580 [ 561.605668] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54cf247f-b2e2-44cf-9625-2c93c47ac8aa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.617559] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c338fc98-ff3e-4009-96a5-2c7ab49109a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.653036] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96e537b1-49da-4d55-a3b0-f78864ac3db5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.664241] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6b55dd5-ad7b-4066-a685-edfa81d83165 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 561.682521] env[62736]: DEBUG nova.compute.provider_tree [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 561.683292] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg d9f2361f1cdd4bd2aaf13733e134975e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 561.693111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9f2361f1cdd4bd2aaf13733e134975e [ 561.777326] env[62736]: INFO nova.scheduler.client.report [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Deleted allocations for instance 2eb46a2a-27fc-4d44-9d18-6d4553026a60 [ 561.788938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 660ba41073f44560bf6d21d7f27c7cda in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 561.825143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 660ba41073f44560bf6d21d7f27c7cda [ 562.045093] env[62736]: DEBUG nova.network.neutron [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 562.045706] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 51297c8534294fc5a6d2c8c28337f60f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.058230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51297c8534294fc5a6d2c8c28337f60f [ 562.193679] env[62736]: DEBUG nova.scheduler.client.report [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 562.196208] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg a4268b4a896c4034ac518ac771f627da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.212415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4268b4a896c4034ac518ac771f627da [ 562.288805] env[62736]: DEBUG oslo_concurrency.lockutils [None req-26cb9926-5a17-47ff-af0e-a897f86c2f17 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "2eb46a2a-27fc-4d44-9d18-6d4553026a60" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 35.373s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.288805] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg da71ba570d834d49ba2044c8edf4d65c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.344811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da71ba570d834d49ba2044c8edf4d65c [ 562.549257] env[62736]: INFO nova.compute.manager [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] [instance: 0b211be4-1955-456d-bf30-5195b5169807] Took 1.03 seconds to deallocate network for instance. [ 562.551082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 9b147f3f5bb741c1a763f334478f23e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.603584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b147f3f5bb741c1a763f334478f23e9 [ 562.700194] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.349s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.700782] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 562.702726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 0a38f18203bc4d4194a0419ab380ffe1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.703395] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 9.746s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.708194] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg c714bf862fce4ec79443037b142052d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.746924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c714bf862fce4ec79443037b142052d2 [ 562.757987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a38f18203bc4d4194a0419ab380ffe1 [ 562.799491] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 562.801577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 15678ae520064a3b970c886cbeb5f77b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 562.842850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15678ae520064a3b970c886cbeb5f77b [ 563.055974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 8fc5087bd0cb42b7b7931c4aa3ad6fba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.096765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fc5087bd0cb42b7b7931c4aa3ad6fba [ 563.208949] env[62736]: DEBUG nova.compute.utils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 563.208949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 9262beb6a85b48e0814539125d8caf3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.210568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 47f873ebeeaa4902bfc1a86e52af4810 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.211428] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 563.211585] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 563.220623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9262beb6a85b48e0814539125d8caf3a [ 563.233949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47f873ebeeaa4902bfc1a86e52af4810 [ 563.288568] env[62736]: DEBUG nova.policy [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a9a0e0bbae8046a783b171c8a4ee3c11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f9f409d646e3429cac4cd3555a5073c2', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 563.322415] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 563.602678] env[62736]: INFO nova.scheduler.client.report [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Deleted allocations for instance 0b211be4-1955-456d-bf30-5195b5169807 [ 563.605717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Expecting reply to msg 65426dc77bb44bc28bfd6f0f516ca3c2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.628842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65426dc77bb44bc28bfd6f0f516ca3c2 [ 563.712996] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 563.714105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg a34219d08a1b4251bdb375cb35922c1f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.749008] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 563.749160] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance ae1041c6-888a-4cb3-8da3-00e43b5b6531 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 563.749286] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance aa9671f3-07a8-422f-93b5-006de0f2cb3c actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 563.749414] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance a5a690d3-8018-4cc5-8d59-55062a595ed0 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 563.749539] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 01b348db-6a0d-4283-b9d9-8edacfd1c219 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 256, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 563.750141] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg d5190e962c604b1f888ee624d7d4c164 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 563.768536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a34219d08a1b4251bdb375cb35922c1f [ 563.777341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5190e962c604b1f888ee624d7d4c164 [ 564.108764] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e57d895d-64f7-4140-add7-4f167bcf3cf9 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447 tempest-FloatingIPsAssociationNegativeTestJSON-1846187447-project-member] Lock "0b211be4-1955-456d-bf30-5195b5169807" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 35.233s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 564.108764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg b44b7ee73191408a85b89f6bf917e0a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 564.135924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b44b7ee73191408a85b89f6bf917e0a1 [ 564.182765] env[62736]: DEBUG nova.compute.manager [None req-7da87e6f-11d0-4817-bd48-aa8be9dca673 tempest-ServerDiagnosticsV248Test-134475782 tempest-ServerDiagnosticsV248Test-134475782-project-admin] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 564.182765] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a64bc65-7870-4a9c-9de0-163d886f41fa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.188859] env[62736]: INFO nova.compute.manager [None req-7da87e6f-11d0-4817-bd48-aa8be9dca673 tempest-ServerDiagnosticsV248Test-134475782 tempest-ServerDiagnosticsV248Test-134475782-project-admin] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Retrieving diagnostics [ 564.189808] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1478e73a-ccbd-4df2-8fe3-79249788a97d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.228858] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 27e299b1c67e43d5a51542c36813974d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 564.254067] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 958e85c0-cb75-4dfd-92c4-b68957e87ec5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 564.254895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg d6948f962c324e38bb7d577b95cece03 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 564.266183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6948f962c324e38bb7d577b95cece03 [ 564.276016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27e299b1c67e43d5a51542c36813974d [ 564.416743] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Successfully created port: d3ab0149-beaf-4cc6-a12c-69e4bba83062 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 564.448871] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquiring lock "a3182b4a-1846-4766-8988-aab0a74fa4c8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 564.448963] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "a3182b4a-1846-4766-8988-aab0a74fa4c8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 564.614017] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 564.615881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg ca29592e610349b0a39154d2f0509b37 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 564.680291] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca29592e610349b0a39154d2f0509b37 [ 564.734795] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 564.758024] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 52f8e827-3faf-47c1-b793-ccacc5a0bc4c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 564.758654] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b23914ff5731491f97085b69cf6965b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 564.765250] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:19Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='84',id=12,is_public=True,memory_mb=256,name='m1.micro',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 564.765506] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 564.765674] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 564.765861] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 564.766470] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 564.766470] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 564.766470] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 564.766613] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 564.766705] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 564.766911] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 564.767112] env[62736]: DEBUG nova.virt.hardware [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 564.768063] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0ee5fbe-698f-4a94-870c-9ede7824222d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 564.771062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b23914ff5731491f97085b69cf6965b4 [ 564.777860] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd7459e2-b3f1-4883-9dd0-c89c7ec91701 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 565.137969] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.263169] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance f537a3a9-b63a-4233-968a-942aca6ada0e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 565.263498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 050d0784fd8542b295057c65b31306f6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 565.275926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 050d0784fd8542b295057c65b31306f6 [ 565.445607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg f02dd21aba814185a381953916f0c31f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 565.456981] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f02dd21aba814185a381953916f0c31f [ 565.766353] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance bd41759b-5c32-42e6-b625-4ea48f98045e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 565.767039] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 48e57b4d8c2840d5a881e391a8461c8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 565.780055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48e57b4d8c2840d5a881e391a8461c8b [ 565.847858] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "c97b25ac-527f-404e-a539-1c0f43bcd66b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.848291] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "c97b25ac-527f-404e-a539-1c0f43bcd66b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 565.950452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.950452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 565.950452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 565.950452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 565.950752] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 565.953183] env[62736]: INFO nova.compute.manager [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Terminating instance [ 565.954932] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "refresh_cache-5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 565.954932] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquired lock "refresh_cache-5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 565.955101] env[62736]: DEBUG nova.network.neutron [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 565.955516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 0783ef14f0b14efc8eb623195247e62e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 565.962164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0783ef14f0b14efc8eb623195247e62e [ 566.270247] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 25508ff0-bc62-47b3-ae9e-7d9d8c735edc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 566.270928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg abe6fd500d3e4f039b8a3b0b6a0aafa5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 566.286202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abe6fd500d3e4f039b8a3b0b6a0aafa5 [ 566.512384] env[62736]: DEBUG nova.network.neutron [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 566.748910] env[62736]: DEBUG nova.network.neutron [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 566.749489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg d3adc6455f14471099bde4078c8de8b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 566.758893] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3adc6455f14471099bde4078c8de8b9 [ 566.774070] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 918aa256-a10d-46d1-b9a4-ecd70cf3566f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 566.774657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 5e68e3129d9f43a48ab335483339b0f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 566.800824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e68e3129d9f43a48ab335483339b0f4 [ 566.969698] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 566.969872] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 567.251990] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Releasing lock "refresh_cache-5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 567.252483] env[62736]: DEBUG nova.compute.manager [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 567.252673] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 567.253557] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e07a5ed-ebdd-46df-8709-02cb33183046 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.265809] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 567.265809] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-d28ba8cf-2fcc-4a8c-9ffe-80958b642171 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.272581] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 567.272581] env[62736]: value = "task-397677" [ 567.272581] env[62736]: _type = "Task" [ 567.272581] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 567.276889] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 64ee370c-6d43-4e2b-a27a-3aa716fdc322 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 567.277576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 533def8eb4c54f37b8d95cd2485930b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 567.282861] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397677, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 567.289561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 533def8eb4c54f37b8d95cd2485930b9 [ 567.782874] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397677, 'name': PowerOffVM_Task, 'duration_secs': 0.135673} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 567.783548] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 3a848010-382b-4753-8420-37222963e7e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 567.784118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 895fa596c3b5443b9c51c0274cdc9033 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 567.785073] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 567.785403] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 567.785706] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-435a81dc-f5ec-427a-83c2-36ac53f308cd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.806796] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 895fa596c3b5443b9c51c0274cdc9033 [ 567.817815] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 567.818020] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 567.818193] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleting the datastore file [datastore1] 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 567.818444] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d31f75fc-7b27-4023-8e49-e2ba92fbee39 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 567.826121] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for the task: (returnval){ [ 567.826121] env[62736]: value = "task-397679" [ 567.826121] env[62736]: _type = "Task" [ 567.826121] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 567.835311] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397679, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 568.286818] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance c7670280-b5a7-4de2-8cf9-6814a81eb1a6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 568.287471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 6ade5669153a45118e5fa642f238ebe0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 568.298405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ade5669153a45118e5fa642f238ebe0 [ 568.338506] env[62736]: DEBUG oslo_vmware.api [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Task: {'id': task-397679, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.098548} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 568.338879] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 568.339167] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 568.339485] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 568.339773] env[62736]: INFO nova.compute.manager [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Took 1.09 seconds to destroy the instance on the hypervisor. [ 568.340134] env[62736]: DEBUG oslo.service.loopingcall [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 568.340514] env[62736]: DEBUG nova.compute.manager [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 568.340724] env[62736]: DEBUG nova.network.neutron [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 568.421468] env[62736]: DEBUG nova.network.neutron [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 568.421468] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6c11e0c8cf074e1d96f29a86ce3f7187 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 568.435514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c11e0c8cf074e1d96f29a86ce3f7187 [ 568.793479] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 558058e0-9808-415a-8866-75d1c8598d43 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 568.793479] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 568.793479] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1536MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 568.924240] env[62736]: DEBUG nova.network.neutron [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 568.924702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bb38dbbaaf2c4a4f8f289358da4efdf6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 568.936579] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb38dbbaaf2c4a4f8f289358da4efdf6 [ 569.145313] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55049096-25f9-4bb6-b3ad-3ba0a1d2e32b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 569.154168] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0450230-abbe-42f1-99a4-c85a30f27152 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 569.193605] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b068b43c-ee8d-4359-9597-32afb27459a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 569.202690] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95a18ab2-9b6b-4651-8b7b-c631ee68f3bb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 569.231872] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 569.232437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 7943972293cb40b68c076ebea37d8ec1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 569.241673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7943972293cb40b68c076ebea37d8ec1 [ 569.427531] env[62736]: INFO nova.compute.manager [-] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Took 1.09 seconds to deallocate network for instance. [ 569.431889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 7f1b26fd4d2b4cfbb221358a51586194 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 569.514482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f1b26fd4d2b4cfbb221358a51586194 [ 569.628911] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquiring lock "4e2f1fc0-102f-4d39-bec2-a2651918a202" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 569.629152] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "4e2f1fc0-102f-4d39-bec2-a2651918a202" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 569.735290] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 569.737615] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 6d194e5710d44facb5438f4bcdb6ac13 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 569.758513] env[62736]: DEBUG nova.compute.manager [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Received event network-changed-d3ab0149-beaf-4cc6-a12c-69e4bba83062 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 569.758702] env[62736]: DEBUG nova.compute.manager [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Refreshing instance network info cache due to event network-changed-d3ab0149-beaf-4cc6-a12c-69e4bba83062. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 569.758906] env[62736]: DEBUG oslo_concurrency.lockutils [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] Acquiring lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 569.759037] env[62736]: DEBUG oslo_concurrency.lockutils [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] Acquired lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 569.759190] env[62736]: DEBUG nova.network.neutron [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Refreshing network info cache for port d3ab0149-beaf-4cc6-a12c-69e4bba83062 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 569.759658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] Expecting reply to msg e15d606f7adc44a3b3e2f9e2a236ed4f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 569.763145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d194e5710d44facb5438f4bcdb6ac13 [ 569.776456] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e15d606f7adc44a3b3e2f9e2a236ed4f [ 569.835290] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "13247e87-08ab-434e-b091-8b6116280fb7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 569.835530] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 569.934698] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 570.236839] env[62736]: ERROR nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 570.236839] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 570.236839] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 570.236839] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 570.236839] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 570.236839] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 570.236839] env[62736]: ERROR nova.compute.manager raise self.value [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 570.236839] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 570.236839] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 570.236839] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 570.237688] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 570.237688] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 570.237688] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 570.237688] env[62736]: ERROR nova.compute.manager [ 570.237688] env[62736]: Traceback (most recent call last): [ 570.237688] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 570.237688] env[62736]: listener.cb(fileno) [ 570.237688] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 570.237688] env[62736]: result = function(*args, **kwargs) [ 570.237688] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 570.237688] env[62736]: return func(*args, **kwargs) [ 570.237688] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 570.237688] env[62736]: raise e [ 570.237688] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 570.237688] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 570.237688] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 570.237688] env[62736]: created_port_ids = self._update_ports_for_instance( [ 570.237688] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 570.237688] env[62736]: with excutils.save_and_reraise_exception(): [ 570.237688] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 570.237688] env[62736]: self.force_reraise() [ 570.237688] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 570.237688] env[62736]: raise self.value [ 570.237688] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 570.237688] env[62736]: updated_port = self._update_port( [ 570.237688] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 570.237688] env[62736]: _ensure_no_port_binding_failure(port) [ 570.237688] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 570.237688] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 570.238629] env[62736]: nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 570.238629] env[62736]: Removing descriptor: 19 [ 570.238629] env[62736]: ERROR nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Traceback (most recent call last): [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] yield resources [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.driver.spawn(context, instance, image_meta, [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self._vmops.spawn(context, instance, image_meta, injected_files, [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 570.238629] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] vm_ref = self.build_virtual_machine(instance, [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] vif_infos = vmwarevif.get_vif_info(self._session, [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] for vif in network_info: [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self._sync_wrapper(fn, *args, **kwargs) [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.wait() [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self[:] = self._gt.wait() [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self._exit_event.wait() [ 570.239028] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] result = hub.switch() [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self.greenlet.switch() [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] result = function(*args, **kwargs) [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return func(*args, **kwargs) [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise e [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] nwinfo = self.network_api.allocate_for_instance( [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 570.239583] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] created_port_ids = self._update_ports_for_instance( [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] with excutils.save_and_reraise_exception(): [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.force_reraise() [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise self.value [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] updated_port = self._update_port( [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] _ensure_no_port_binding_failure(port) [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 570.239940] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise exception.PortBindingFailed(port_id=port['id']) [ 570.240409] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 570.240409] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] [ 570.240409] env[62736]: INFO nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Terminating instance [ 570.240409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 570.240409] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 570.240409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 7.537s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 570.241448] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.450s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 570.244032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg beab841b680b4f9f9eb944848da0634e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 570.288486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg beab841b680b4f9f9eb944848da0634e [ 570.321505] env[62736]: DEBUG nova.network.neutron [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 570.913010] env[62736]: DEBUG nova.network.neutron [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 570.913458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] Expecting reply to msg 786b6657f60241869c85e60c5dd8a9ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 570.931475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 786b6657f60241869c85e60c5dd8a9ce [ 571.051462] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f3bc0c8-3f2a-4c41-b72d-b3168798ce99 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.058131] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-077331e8-464f-4a63-a075-10d974c24b21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.094505] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f39be86-ca60-4c66-9040-a2e117132a72 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.103141] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd1bbb2e-84d8-4661-8abd-399fc718d70e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 571.137909] env[62736]: DEBUG nova.compute.provider_tree [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 571.137909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 9892fd47125b47c2afbd520e0718b732 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 571.154487] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9892fd47125b47c2afbd520e0718b732 [ 571.416958] env[62736]: DEBUG oslo_concurrency.lockutils [req-adf5d4e8-f285-4731-9921-2b2baeb29aa8 req-0a87ac6a-d185-4df8-8786-4fb235ab7953 service nova] Releasing lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 571.416958] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 571.416958] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 571.416958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 22fb000d3fc64e2fa3e1db5008fc6943 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 571.423580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22fb000d3fc64e2fa3e1db5008fc6943 [ 571.641970] env[62736]: DEBUG nova.scheduler.client.report [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 571.641970] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 9a15f350bf0647b99de5dfdf09da753e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 571.656189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a15f350bf0647b99de5dfdf09da753e [ 572.069966] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 572.143846] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.903s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.144518] env[62736]: ERROR nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Traceback (most recent call last): [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.driver.spawn(context, instance, image_meta, [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self._vmops.spawn(context, instance, image_meta, injected_files, [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] vm_ref = self.build_virtual_machine(instance, [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] vif_infos = vmwarevif.get_vif_info(self._session, [ 572.144518] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] for vif in network_info: [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self._sync_wrapper(fn, *args, **kwargs) [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.wait() [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self[:] = self._gt.wait() [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self._exit_event.wait() [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] result = hub.switch() [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 572.144907] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return self.greenlet.switch() [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] result = function(*args, **kwargs) [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] return func(*args, **kwargs) [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise e [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] nwinfo = self.network_api.allocate_for_instance( [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] created_port_ids = self._update_ports_for_instance( [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] with excutils.save_and_reraise_exception(): [ 572.145283] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] self.force_reraise() [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise self.value [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] updated_port = self._update_port( [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] _ensure_no_port_binding_failure(port) [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] raise exception.PortBindingFailed(port_id=port['id']) [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] nova.exception.PortBindingFailed: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. [ 572.145652] env[62736]: ERROR nova.compute.manager [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] [ 572.145982] env[62736]: DEBUG nova.compute.utils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 572.146447] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.917s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.148250] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1222019ae52148058b000d303bf62409 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 572.163086] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Build of instance ae1041c6-888a-4cb3-8da3-00e43b5b6531 was re-scheduled: Binding failed for port 374282b5-a930-4685-a3e9-c381d14b89c8, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 572.163086] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 572.163086] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 572.163086] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 572.163383] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 572.163383] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 6ce6d0aff3c4449ba654a7fea6f422b1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 572.182578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ce6d0aff3c4449ba654a7fea6f422b1 [ 572.204825] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1222019ae52148058b000d303bf62409 [ 572.519855] env[62736]: DEBUG nova.compute.manager [req-ab789b23-4cc1-4454-ac1b-8cee610ca4a8 req-d2248fa4-033a-49f2-a72f-781e185c7479 service nova] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Received event network-vif-deleted-d3ab0149-beaf-4cc6-a12c-69e4bba83062 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 572.700738] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 572.958903] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cd519e5-5aee-4281-8c68-200a060cfb72 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 572.964637] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9aa608e7-5483-42b6-af4d-69ffc68faeb1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.006565] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 573.007360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg df869ce3f5684005b7009fada7372498 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.008984] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 573.009607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 0bbb2390ee29443fba4ab2e777701cbe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.011131] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e9829aa-80e1-4c2e-9e7b-d857b8a7f1ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.019014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df869ce3f5684005b7009fada7372498 [ 573.019751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bbb2390ee29443fba4ab2e777701cbe [ 573.021645] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecd2c6fe-0475-4579-8502-31cc5ba05062 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.044783] env[62736]: DEBUG nova.compute.provider_tree [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 573.045510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 517de6308ef0454a90cc7369aa64f68e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.052496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 517de6308ef0454a90cc7369aa64f68e [ 573.515568] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 573.516062] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 573.516239] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 573.516946] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-ae1041c6-888a-4cb3-8da3-00e43b5b6531" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 573.516946] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 573.517065] env[62736]: DEBUG nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 573.517185] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 573.518923] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-356b30a8-60d9-491d-a4f2-62b97a320b01 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.527597] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49da5df9-54aa-41b0-956c-157782e017b5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.548160] env[62736]: DEBUG nova.scheduler.client.report [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 573.550737] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg e56c01cbd72e404eb651693dcbab45b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.556165] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 01b348db-6a0d-4283-b9d9-8edacfd1c219 could not be found. [ 573.556375] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 573.556544] env[62736]: INFO nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Took 0.04 seconds to destroy the instance on the hypervisor. [ 573.556778] env[62736]: DEBUG oslo.service.loopingcall [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 573.557522] env[62736]: DEBUG nova.compute.manager [-] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 573.557618] env[62736]: DEBUG nova.network.neutron [-] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 573.565311] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e56c01cbd72e404eb651693dcbab45b2 [ 573.618694] env[62736]: DEBUG nova.network.neutron [-] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 573.619538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2f3be282a1b44326a61368cc74f6885e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.627113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f3be282a1b44326a61368cc74f6885e [ 573.631587] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 573.632189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 79a44565af1847ddb4352abe2c82ee6b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 573.638678] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79a44565af1847ddb4352abe2c82ee6b [ 574.058407] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.912s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 574.059057] env[62736]: ERROR nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Traceback (most recent call last): [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.driver.spawn(context, instance, image_meta, [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] vm_ref = self.build_virtual_machine(instance, [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] vif_infos = vmwarevif.get_vif_info(self._session, [ 574.059057] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] for vif in network_info: [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self._sync_wrapper(fn, *args, **kwargs) [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.wait() [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self[:] = self._gt.wait() [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self._exit_event.wait() [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] result = hub.switch() [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 574.059445] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return self.greenlet.switch() [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] result = function(*args, **kwargs) [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] return func(*args, **kwargs) [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise e [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] nwinfo = self.network_api.allocate_for_instance( [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] created_port_ids = self._update_ports_for_instance( [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] with excutils.save_and_reraise_exception(): [ 574.059809] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] self.force_reraise() [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise self.value [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] updated_port = self._update_port( [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] _ensure_no_port_binding_failure(port) [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] raise exception.PortBindingFailed(port_id=port['id']) [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] nova.exception.PortBindingFailed: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. [ 574.060171] env[62736]: ERROR nova.compute.manager [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] [ 574.060473] env[62736]: DEBUG nova.compute.utils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 574.061107] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.088s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 574.062566] env[62736]: INFO nova.compute.claims [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 574.064057] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 957f9a1df18e4d9984c12a61157669c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.072661] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Build of instance aa9671f3-07a8-422f-93b5-006de0f2cb3c was re-scheduled: Binding failed for port ea932e45-f256-494c-9fc5-5ace73092f2f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 574.073081] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 574.073345] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 574.073525] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 574.073728] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 574.074060] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0c1974620dbf46ed9ae26c859f61e0b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.088760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c1974620dbf46ed9ae26c859f61e0b4 [ 574.112285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 957f9a1df18e4d9984c12a61157669c9 [ 574.122636] env[62736]: DEBUG nova.network.neutron [-] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 574.123163] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7c4378b88582442ea03940a86f298e31 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.134470] env[62736]: DEBUG nova.network.neutron [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 574.135055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 99cc6799e8c240fe8f1fa1543e5391d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.142911] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c4378b88582442ea03940a86f298e31 [ 574.143509] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99cc6799e8c240fe8f1fa1543e5391d6 [ 574.577820] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 4f6c4f5df701492d8579ea4608b90574 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.585831] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f6c4f5df701492d8579ea4608b90574 [ 574.625410] env[62736]: INFO nova.compute.manager [-] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Took 1.07 seconds to deallocate network for instance. [ 574.629218] env[62736]: DEBUG nova.compute.claims [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 574.629320] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 574.636981] env[62736]: INFO nova.compute.manager [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: ae1041c6-888a-4cb3-8da3-00e43b5b6531] Took 1.12 seconds to deallocate network for instance. [ 574.638750] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg be5cb5b827a7429b8b99f6ff1180bcf4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 574.702336] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be5cb5b827a7429b8b99f6ff1180bcf4 [ 574.715337] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 574.913318] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquiring lock "ce17af27-45c3-461e-8cb7-b9b023d88361" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 574.913569] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "ce17af27-45c3-461e-8cb7-b9b023d88361" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.146797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 915869d977834f1da62dc8d292829019 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 575.191503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 915869d977834f1da62dc8d292829019 [ 575.251240] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 575.251240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 12df7f4115d341888135488de30965a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 575.269860] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12df7f4115d341888135488de30965a3 [ 575.419637] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7a411e1-c9f4-40ce-9d25-81542581f9de {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.428901] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68f28ca9-f088-4807-a93e-9918120efcb5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.465534] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8caf6819-82c6-4649-b609-ed1c8e85528e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.474562] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6d430c1-5e56-4164-8859-2fe2f7053e89 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 575.489575] env[62736]: DEBUG nova.compute.provider_tree [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 575.490168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg e1617ab726194cbab0340e607c1a752f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 575.498296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1617ab726194cbab0340e607c1a752f [ 575.671660] env[62736]: INFO nova.scheduler.client.report [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Deleted allocations for instance ae1041c6-888a-4cb3-8da3-00e43b5b6531 [ 575.678836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg e38afa417cd04de7babea7d7f0b7be4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 575.697274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e38afa417cd04de7babea7d7f0b7be4d [ 575.754547] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-aa9671f3-07a8-422f-93b5-006de0f2cb3c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 575.754793] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 575.754976] env[62736]: DEBUG nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 575.755140] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 575.832413] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 575.832999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6ea0bd92d1754786bca3714dddab49e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 575.839529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ea0bd92d1754786bca3714dddab49e8 [ 575.993451] env[62736]: DEBUG nova.scheduler.client.report [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 575.997331] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 5c7d2e7015454ba09f8f1e37bd8d9bbf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.025595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c7d2e7015454ba09f8f1e37bd8d9bbf [ 576.181040] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d95615ff-60d5-405e-97b1-3e2c394f2151 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "ae1041c6-888a-4cb3-8da3-00e43b5b6531" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 46.575s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.181436] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 2892207c327445e2a6bf04da76227b71 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.194183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2892207c327445e2a6bf04da76227b71 [ 576.335434] env[62736]: DEBUG nova.network.neutron [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 576.335983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 5b4a754119ac4fefa81c902fda701dee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.345050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b4a754119ac4fefa81c902fda701dee [ 576.512298] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.451s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.512783] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 576.514634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg a72e041d832f40ff8345472b17993452 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.515383] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 20.316s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.518320] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 191f944f2c6e4106b93fadbf7093fe9c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.551206] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a72e041d832f40ff8345472b17993452 [ 576.561772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 191f944f2c6e4106b93fadbf7093fe9c [ 576.683723] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 576.685845] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 3038d13eb2c1433daf24d9298509478f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.730492] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3038d13eb2c1433daf24d9298509478f [ 576.845552] env[62736]: INFO nova.compute.manager [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: aa9671f3-07a8-422f-93b5-006de0f2cb3c] Took 1.09 seconds to deallocate network for instance. [ 576.847331] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 330b6dbbdf3a428d9744d47011755b25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 576.899724] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 330b6dbbdf3a428d9744d47011755b25 [ 577.036549] env[62736]: DEBUG nova.compute.utils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 577.036549] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 20a57f54689a428fbd30ccc25d941ac1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.041091] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 577.041307] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 577.047989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20a57f54689a428fbd30ccc25d941ac1 [ 577.204332] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.308770] env[62736]: DEBUG nova.policy [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6132ab51d5b34a8c9c21a6aa642bc15d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '203ab2a2f81f449e81ec2fdeb589380a', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 577.352048] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 20632da8a5424feaba5cd470dadbb08f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.358960] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d4d1bff-f846-4c66-bb75-0e925096739b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.367436] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e3b5e54-5474-4b18-8f7e-f2bdb1d5e5f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.409893] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20632da8a5424feaba5cd470dadbb08f [ 577.410911] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ab50c5b-9f45-4732-b410-42cf9dba9a23 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.423171] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-173113a4-a5dd-4e5f-b458-740295b36e55 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 577.438699] env[62736]: DEBUG nova.compute.provider_tree [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 577.439737] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 6bd56c50cc4a4f42b6e5672af864c239 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.448920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bd56c50cc4a4f42b6e5672af864c239 [ 577.544344] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 577.546765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 8a3393ea448d42dd9e954badb92bd00d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.588989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a3393ea448d42dd9e954badb92bd00d [ 577.879753] env[62736]: INFO nova.scheduler.client.report [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance aa9671f3-07a8-422f-93b5-006de0f2cb3c [ 577.885773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg e9bfc3539d514c93b76306b59d6d8b32 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.932591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9bfc3539d514c93b76306b59d6d8b32 [ 577.941591] env[62736]: DEBUG nova.scheduler.client.report [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 577.943987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 284d5e7927ec452e8381a0161c0b7792 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 577.973840] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 284d5e7927ec452e8381a0161c0b7792 [ 578.024322] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquiring lock "9a445d5c-7a5f-4149-b198-306a04b4c040" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.024546] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "9a445d5c-7a5f-4149-b198-306a04b4c040" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.055933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg a86be61ec08e42439190c0e45f5227a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.088695] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a86be61ec08e42439190c0e45f5227a9 [ 578.331287] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Successfully created port: 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 578.394579] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a5c29c37-c3c6-492e-b681-b76ce126536f tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "aa9671f3-07a8-422f-93b5-006de0f2cb3c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 47.849s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.395199] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg ef18dde092f649a0b86bfe40de1189ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.404829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef18dde092f649a0b86bfe40de1189ef [ 578.448057] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.931s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.448057] env[62736]: ERROR nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Traceback (most recent call last): [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.driver.spawn(context, instance, image_meta, [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 578.448057] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] vm_ref = self.build_virtual_machine(instance, [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] vif_infos = vmwarevif.get_vif_info(self._session, [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] for vif in network_info: [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self._sync_wrapper(fn, *args, **kwargs) [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.wait() [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self[:] = self._gt.wait() [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self._exit_event.wait() [ 578.448403] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] result = hub.switch() [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return self.greenlet.switch() [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] result = function(*args, **kwargs) [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] return func(*args, **kwargs) [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise e [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] nwinfo = self.network_api.allocate_for_instance( [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 578.448819] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] created_port_ids = self._update_ports_for_instance( [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] with excutils.save_and_reraise_exception(): [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] self.force_reraise() [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise self.value [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] updated_port = self._update_port( [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] _ensure_no_port_binding_failure(port) [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 578.449240] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] raise exception.PortBindingFailed(port_id=port['id']) [ 578.449699] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] nova.exception.PortBindingFailed: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. [ 578.449699] env[62736]: ERROR nova.compute.manager [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] [ 578.449699] env[62736]: DEBUG nova.compute.utils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 578.450770] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.889s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.451311] env[62736]: INFO nova.compute.claims [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 578.452844] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 8f5d890ada7246a1a8e7efdfaf576238 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.453939] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Build of instance a5a690d3-8018-4cc5-8d59-55062a595ed0 was re-scheduled: Binding failed for port 42afc0e2-7da7-431c-a861-5c77b0a53036, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 578.454365] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 578.454577] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 578.454712] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquired lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 578.454860] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 578.455204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 0b65612bfd874213be7924f1a8c5945a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.468208] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b65612bfd874213be7924f1a8c5945a [ 578.504806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f5d890ada7246a1a8e7efdfaf576238 [ 578.528649] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 578.558600] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 578.583733] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 578.583827] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 578.583982] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 578.584247] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 578.584404] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 578.585182] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 578.585433] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 578.585599] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 578.585763] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 578.585924] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 578.586096] env[62736]: DEBUG nova.virt.hardware [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 578.586937] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91b987ac-67c9-4e5a-869d-386f6f0367ea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 578.596139] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-445403f7-01eb-4487-841a-9695677ace0f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 578.721529] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 578.722392] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg e5a755b3f6644680aa066573b7b2b43e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.731347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5a755b3f6644680aa066573b7b2b43e [ 578.896981] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 578.898718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 983ed0beb24b49fc975867be9133189a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.921360] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "ad008294-52db-40de-95ad-3e0f2d30a462" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.921602] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "ad008294-52db-40de-95ad-3e0f2d30a462" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.953166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 983ed0beb24b49fc975867be9133189a [ 578.967912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 67819c0f9a3041bcb5ff1ad95181c552 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 578.975715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67819c0f9a3041bcb5ff1ad95181c552 [ 579.224313] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Releasing lock "refresh_cache-a5a690d3-8018-4cc5-8d59-55062a595ed0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 579.224557] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 579.224862] env[62736]: DEBUG nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 579.224956] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 579.239892] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 579.240514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 406c74812b544436b2172a0037a86280 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 579.247087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 406c74812b544436b2172a0037a86280 [ 579.419768] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 579.742468] env[62736]: DEBUG nova.network.neutron [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 579.743137] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 0451db7a2f5844498abc1e291b29916f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 579.751498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0451db7a2f5844498abc1e291b29916f [ 579.799169] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe974bb2-1377-483f-9474-2b62a380a0dd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.808140] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-273e5344-bf01-47de-b90b-8139852e8af4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.840337] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a384b6a6-e923-4c23-a78d-ebbbccb6596f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.849063] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f336e07d-6df7-4601-8b31-7cc5c2d69275 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 579.864200] env[62736]: DEBUG nova.compute.provider_tree [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 579.864861] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg c48167c54ead4179a99c6104c4495b75 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 579.873611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c48167c54ead4179a99c6104c4495b75 [ 580.246082] env[62736]: INFO nova.compute.manager [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: a5a690d3-8018-4cc5-8d59-55062a595ed0] Took 1.02 seconds to deallocate network for instance. [ 580.247167] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 911dfc156ddc43fbacb6eb71c003d05d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 580.325230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 911dfc156ddc43fbacb6eb71c003d05d [ 580.367922] env[62736]: DEBUG nova.scheduler.client.report [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 580.370495] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 804aace5f34f4392883f984f040730b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 580.387246] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 804aace5f34f4392883f984f040730b3 [ 580.752503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg b4aea3464ae34609bb3de5313582e804 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 580.816029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4aea3464ae34609bb3de5313582e804 [ 580.873675] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.424s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.874227] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 580.878229] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg e174dcbd76ce434c812718fe840079b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 580.878229] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.564s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.878890] env[62736]: INFO nova.compute.claims [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 580.880507] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg e20d6d9669224b49bfa9c907e0cfea6e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 580.891523] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquiring lock "38767b46-9a31-4031-981b-424497dc4a34" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.891523] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "38767b46-9a31-4031-981b-424497dc4a34" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.952085] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e174dcbd76ce434c812718fe840079b8 [ 580.959059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e20d6d9669224b49bfa9c907e0cfea6e [ 581.206794] env[62736]: ERROR nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 581.206794] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 581.206794] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 581.206794] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 581.206794] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 581.206794] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 581.206794] env[62736]: ERROR nova.compute.manager raise self.value [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 581.206794] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 581.206794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 581.206794] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 581.207272] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 581.207272] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 581.207272] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 581.207272] env[62736]: ERROR nova.compute.manager [ 581.207272] env[62736]: Traceback (most recent call last): [ 581.207272] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 581.207272] env[62736]: listener.cb(fileno) [ 581.207272] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 581.207272] env[62736]: result = function(*args, **kwargs) [ 581.207272] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 581.207272] env[62736]: return func(*args, **kwargs) [ 581.207272] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 581.207272] env[62736]: raise e [ 581.207272] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 581.207272] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 581.207272] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 581.207272] env[62736]: created_port_ids = self._update_ports_for_instance( [ 581.207272] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 581.207272] env[62736]: with excutils.save_and_reraise_exception(): [ 581.207272] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 581.207272] env[62736]: self.force_reraise() [ 581.207272] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 581.207272] env[62736]: raise self.value [ 581.207272] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 581.207272] env[62736]: updated_port = self._update_port( [ 581.207272] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 581.207272] env[62736]: _ensure_no_port_binding_failure(port) [ 581.207272] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 581.207272] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 581.208090] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 581.208090] env[62736]: Removing descriptor: 19 [ 581.208090] env[62736]: ERROR nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Traceback (most recent call last): [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] yield resources [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.driver.spawn(context, instance, image_meta, [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 581.208090] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] vm_ref = self.build_virtual_machine(instance, [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] vif_infos = vmwarevif.get_vif_info(self._session, [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] for vif in network_info: [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self._sync_wrapper(fn, *args, **kwargs) [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.wait() [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self[:] = self._gt.wait() [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self._exit_event.wait() [ 581.208525] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] result = hub.switch() [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self.greenlet.switch() [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] result = function(*args, **kwargs) [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return func(*args, **kwargs) [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise e [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] nwinfo = self.network_api.allocate_for_instance( [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 581.208889] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] created_port_ids = self._update_ports_for_instance( [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] with excutils.save_and_reraise_exception(): [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.force_reraise() [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise self.value [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] updated_port = self._update_port( [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] _ensure_no_port_binding_failure(port) [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 581.209252] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise exception.PortBindingFailed(port_id=port['id']) [ 581.209641] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 581.209641] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] [ 581.209641] env[62736]: INFO nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Terminating instance [ 581.210808] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquiring lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 581.210965] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquired lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 581.211122] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 581.211540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 8b40e922c3c64aa5a0f6ef3244b6fb9e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.222762] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b40e922c3c64aa5a0f6ef3244b6fb9e [ 581.285709] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquiring lock "6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 581.285959] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 581.298706] env[62736]: INFO nova.scheduler.client.report [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Deleted allocations for instance a5a690d3-8018-4cc5-8d59-55062a595ed0 [ 581.305357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 9507c6e914b84c5aa2167ff5774570d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.333062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9507c6e914b84c5aa2167ff5774570d2 [ 581.384917] env[62736]: DEBUG nova.compute.utils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 581.386084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 3c05df2eb9c840dca6350b0451c52973 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.387556] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 581.387851] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 581.393017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 3a7178a5bb7642299b1472d2274b4607 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.407770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a7178a5bb7642299b1472d2274b4607 [ 581.411466] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c05df2eb9c840dca6350b0451c52973 [ 581.640351] env[62736]: DEBUG nova.policy [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a5b16af3d328480fae79eae1aff63f64', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bc7ba68e037a4f28b05c14e7bdbb24e4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 581.752483] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 581.815486] env[62736]: DEBUG oslo_concurrency.lockutils [None req-552db156-4d36-4ab3-b539-a9f13c2fd531 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "a5a690d3-8018-4cc5-8d59-55062a595ed0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 50.380s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 581.815486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg c602095340694fe4ae7ab791aac0c0dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.839930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c602095340694fe4ae7ab791aac0c0dd [ 581.895407] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 581.897198] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 0f7665a46f51450487c433603cb3bd44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 581.940030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f7665a46f51450487c433603cb3bd44 [ 582.010750] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 582.011474] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg e72cd3ea7b184041aefb5153c96e42c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.024442] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e72cd3ea7b184041aefb5153c96e42c8 [ 582.273859] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Successfully created port: e6cc30e4-2174-44e6-a3ca-ec741165ee60 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 582.308561] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-814986e8-acbb-4146-ae62-fc81e8c14056 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.324338] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 582.326693] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg eb9690d89c064ff4823ffd21395b23f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.336281] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4832838b-62d7-456b-b228-d365daf13093 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.340911] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "41650a98-fda7-44fd-aca6-d3b94cf17308" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.341144] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "41650a98-fda7-44fd-aca6-d3b94cf17308" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.374984] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae278563-9d35-4723-bd2f-f145f1efe0fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.384152] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f663515a-f628-47b8-b01b-f0ece3f7121b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.404838] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 83773a6957704a93b2d39dd303d0f77c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.406150] env[62736]: DEBUG nova.compute.provider_tree [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 582.406578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg f3be4ed33ad74b9097559415ac8befc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.407743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb9690d89c064ff4823ffd21395b23f2 [ 582.413377] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3be4ed33ad74b9097559415ac8befc6 [ 582.473008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83773a6957704a93b2d39dd303d0f77c [ 582.514153] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Releasing lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 582.514685] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 582.514770] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 582.515039] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-594f96e2-86c0-4752-abff-c6863dc8dc63 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.524744] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98dc68f5-fc25-4eb9-8a8d-43604e95db05 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.566518] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 958e85c0-cb75-4dfd-92c4-b68957e87ec5 could not be found. [ 582.566740] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 582.566946] env[62736]: INFO nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Took 0.05 seconds to destroy the instance on the hypervisor. [ 582.567143] env[62736]: DEBUG oslo.service.loopingcall [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 582.567352] env[62736]: DEBUG nova.compute.manager [-] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 582.567437] env[62736]: DEBUG nova.network.neutron [-] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 582.705038] env[62736]: DEBUG nova.network.neutron [-] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 582.705566] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d317ec07fbf140cfa1db4ec880782c90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.713744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d317ec07fbf140cfa1db4ec880782c90 [ 582.858200] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.910293] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 582.914428] env[62736]: DEBUG nova.scheduler.client.report [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 582.914958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 35dcd9a2952a42638f51892c5df1807f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 582.933196] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35dcd9a2952a42638f51892c5df1807f [ 582.944780] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 582.945022] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 582.945171] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 582.945353] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 582.945492] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 582.945631] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 582.945824] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 582.945969] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 582.946121] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 582.946270] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 582.946427] env[62736]: DEBUG nova.virt.hardware [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 582.947285] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b29c1bff-f897-4935-8ef2-817ffa29e4f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.957561] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-082af4a2-a23e-4c8f-a712-363ecb9d31a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 583.166361] env[62736]: DEBUG nova.compute.manager [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Received event network-changed-3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 583.166579] env[62736]: DEBUG nova.compute.manager [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Refreshing instance network info cache due to event network-changed-3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 583.166787] env[62736]: DEBUG oslo_concurrency.lockutils [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] Acquiring lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 583.166923] env[62736]: DEBUG oslo_concurrency.lockutils [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] Acquired lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 583.167077] env[62736]: DEBUG nova.network.neutron [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Refreshing network info cache for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 583.167504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] Expecting reply to msg 5a9937159ed343a683b2702210f03b6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.174980] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a9937159ed343a683b2702210f03b6c [ 583.208417] env[62736]: DEBUG nova.network.neutron [-] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 583.208417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 968944722389437e8e04521a75275fd3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.221239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 968944722389437e8e04521a75275fd3 [ 583.418713] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.541s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 583.419242] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 583.420982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg d0f4313493454519bec2e37092d018da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.422274] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.028s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 583.423399] env[62736]: INFO nova.compute.claims [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 583.425073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg fe8e640e764b479abfdc844b0498b33f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.465222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0f4313493454519bec2e37092d018da [ 583.474651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe8e640e764b479abfdc844b0498b33f [ 583.685578] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquiring lock "3e82fa5d-6610-4d3d-972f-4120aca5e8a7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.685578] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "3e82fa5d-6610-4d3d-972f-4120aca5e8a7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 583.695463] env[62736]: DEBUG nova.network.neutron [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 583.710496] env[62736]: INFO nova.compute.manager [-] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Took 1.14 seconds to deallocate network for instance. [ 583.712773] env[62736]: DEBUG nova.compute.claims [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 583.712974] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 583.874023] env[62736]: DEBUG nova.network.neutron [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 583.874552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] Expecting reply to msg 56b6a1c32d0d4a4781abdecaeb6512e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.883275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56b6a1c32d0d4a4781abdecaeb6512e8 [ 583.929536] env[62736]: DEBUG nova.compute.utils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 583.930160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 5ad4a58b38f84931b5cc9b75da0d04c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.932166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg dd7803e52648419e956b1ebd57bdff13 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 583.933096] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 583.933265] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 583.940975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd7803e52648419e956b1ebd57bdff13 [ 583.949307] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ad4a58b38f84931b5cc9b75da0d04c5 [ 584.012345] env[62736]: DEBUG nova.policy [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '064e7b872e9a4d7f9b13a6b722121734', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '51d621f5cd014007a7d2e44bcd714cdf', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 584.363898] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Successfully created port: 3b1dae23-7093-47dd-9cbe-fa5054998b51 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 584.377054] env[62736]: DEBUG oslo_concurrency.lockutils [req-ef0aa6b5-016c-4bf4-9157-5f6c7637fa0e req-df0ad117-5c95-4eb2-9ce0-eb8ec0048fa3 service nova] Releasing lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 584.433823] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 584.435538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 4256ad74fed94e9693312dde2dde941b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 584.473599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4256ad74fed94e9693312dde2dde941b [ 584.762237] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquiring lock "15215512-da0c-407f-a352-43c051698f2d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 584.762883] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "15215512-da0c-407f-a352-43c051698f2d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 584.794098] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0c073bf-df06-4752-b306-91360d6f8d0b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.803662] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e4b9b850-2265-4219-bfe1-8f567d07cb85 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.806517] env[62736]: ERROR nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 584.806517] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 584.806517] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 584.806517] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 584.806517] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 584.806517] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 584.806517] env[62736]: ERROR nova.compute.manager raise self.value [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 584.806517] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 584.806517] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 584.806517] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 584.807057] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 584.807057] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 584.807057] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 584.807057] env[62736]: ERROR nova.compute.manager [ 584.807057] env[62736]: Traceback (most recent call last): [ 584.807057] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 584.807057] env[62736]: listener.cb(fileno) [ 584.807057] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 584.807057] env[62736]: result = function(*args, **kwargs) [ 584.807057] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 584.807057] env[62736]: return func(*args, **kwargs) [ 584.807057] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 584.807057] env[62736]: raise e [ 584.807057] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 584.807057] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 584.807057] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 584.807057] env[62736]: created_port_ids = self._update_ports_for_instance( [ 584.807057] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 584.807057] env[62736]: with excutils.save_and_reraise_exception(): [ 584.807057] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 584.807057] env[62736]: self.force_reraise() [ 584.807057] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 584.807057] env[62736]: raise self.value [ 584.807057] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 584.807057] env[62736]: updated_port = self._update_port( [ 584.807057] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 584.807057] env[62736]: _ensure_no_port_binding_failure(port) [ 584.807057] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 584.807057] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 584.808046] env[62736]: nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 584.808046] env[62736]: Removing descriptor: 19 [ 584.808046] env[62736]: ERROR nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Traceback (most recent call last): [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] yield resources [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.driver.spawn(context, instance, image_meta, [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 584.808046] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] vm_ref = self.build_virtual_machine(instance, [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] vif_infos = vmwarevif.get_vif_info(self._session, [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] for vif in network_info: [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self._sync_wrapper(fn, *args, **kwargs) [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.wait() [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self[:] = self._gt.wait() [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self._exit_event.wait() [ 584.808442] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] result = hub.switch() [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self.greenlet.switch() [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] result = function(*args, **kwargs) [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return func(*args, **kwargs) [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise e [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] nwinfo = self.network_api.allocate_for_instance( [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 584.808868] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] created_port_ids = self._update_ports_for_instance( [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] with excutils.save_and_reraise_exception(): [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.force_reraise() [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise self.value [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] updated_port = self._update_port( [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] _ensure_no_port_binding_failure(port) [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 584.809506] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise exception.PortBindingFailed(port_id=port['id']) [ 584.809900] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 584.809900] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] [ 584.809900] env[62736]: INFO nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Terminating instance [ 584.810129] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquiring lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 584.810129] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquired lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 584.810751] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 584.810751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 1bf1e3ea47fb48feb17b7a1820fe3db7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 584.841482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bf1e3ea47fb48feb17b7a1820fe3db7 [ 584.845105] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d0c84de-b2d2-4a14-8b4f-69471fc0c8d0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.859642] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81f99350-3653-4711-a019-600e0d3cf77f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.864558] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 584.879149] env[62736]: DEBUG nova.compute.provider_tree [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 584.879748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 74eefff14c7648c8a5ab2c1498901ac0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 584.894946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74eefff14c7648c8a5ab2c1498901ac0 [ 584.942795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 6c179fa4ddb74482962656faa59d10da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 584.976532] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c179fa4ddb74482962656faa59d10da [ 584.977970] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 584.978494] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 012f802b4c1a48e0900cfafdb0ea8bdb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 584.987450] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 012f802b4c1a48e0900cfafdb0ea8bdb [ 585.239253] env[62736]: ERROR nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 585.239253] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 585.239253] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 585.239253] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 585.239253] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 585.239253] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 585.239253] env[62736]: ERROR nova.compute.manager raise self.value [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 585.239253] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 585.239253] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 585.239253] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 585.239813] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 585.239813] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 585.239813] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 585.239813] env[62736]: ERROR nova.compute.manager [ 585.239813] env[62736]: Traceback (most recent call last): [ 585.239813] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 585.239813] env[62736]: listener.cb(fileno) [ 585.239813] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 585.239813] env[62736]: result = function(*args, **kwargs) [ 585.239813] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 585.239813] env[62736]: return func(*args, **kwargs) [ 585.239813] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 585.239813] env[62736]: raise e [ 585.239813] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 585.239813] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 585.239813] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 585.239813] env[62736]: created_port_ids = self._update_ports_for_instance( [ 585.239813] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 585.239813] env[62736]: with excutils.save_and_reraise_exception(): [ 585.239813] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 585.239813] env[62736]: self.force_reraise() [ 585.239813] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 585.239813] env[62736]: raise self.value [ 585.239813] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 585.239813] env[62736]: updated_port = self._update_port( [ 585.239813] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 585.239813] env[62736]: _ensure_no_port_binding_failure(port) [ 585.239813] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 585.239813] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 585.240773] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 585.240773] env[62736]: Removing descriptor: 16 [ 585.370015] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "c4512101-c88c-4f3c-bde1-def0f3520101" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.370651] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "c4512101-c88c-4f3c-bde1-def0f3520101" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.386163] env[62736]: DEBUG nova.scheduler.client.report [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 585.399057] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 9b9496cbcb094e1a9810db43feb8e9e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.410717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b9496cbcb094e1a9810db43feb8e9e3 [ 585.445651] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 585.466523] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 585.466800] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 585.466962] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 585.467119] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 585.467299] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 585.467463] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 585.467666] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 585.467818] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 585.468116] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 585.468283] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 585.468452] env[62736]: DEBUG nova.virt.hardware [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 585.469313] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c77d8e14-f4dc-4c3d-955e-6d4040e7e725 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.478738] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-baf84a49-bdce-462f-bdc6-f7f1f22aff02 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.482588] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Releasing lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 585.483012] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 585.483194] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 585.483429] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-764bf4bb-ef74-4242-86a4-59e3f20ee8b3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.495303] env[62736]: ERROR nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Traceback (most recent call last): [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] yield resources [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.driver.spawn(context, instance, image_meta, [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] vm_ref = self.build_virtual_machine(instance, [ 585.495303] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] vif_infos = vmwarevif.get_vif_info(self._session, [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] for vif in network_info: [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return self._sync_wrapper(fn, *args, **kwargs) [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.wait() [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self[:] = self._gt.wait() [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return self._exit_event.wait() [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 585.495743] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] current.throw(*self._exc) [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] result = function(*args, **kwargs) [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return func(*args, **kwargs) [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise e [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] nwinfo = self.network_api.allocate_for_instance( [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] created_port_ids = self._update_ports_for_instance( [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] with excutils.save_and_reraise_exception(): [ 585.496257] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.force_reraise() [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise self.value [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] updated_port = self._update_port( [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] _ensure_no_port_binding_failure(port) [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise exception.PortBindingFailed(port_id=port['id']) [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 585.496682] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] [ 585.496682] env[62736]: INFO nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Terminating instance [ 585.498360] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquiring lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 585.498531] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquired lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 585.498692] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 585.499107] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 0f6d434227d34d189d3aa7ab97d2988c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.502295] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37cbb06e-48f3-49ea-8f7c-0284f312b627 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.512762] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f6d434227d34d189d3aa7ab97d2988c [ 585.526285] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 52f8e827-3faf-47c1-b793-ccacc5a0bc4c could not be found. [ 585.526488] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 585.526655] env[62736]: INFO nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 585.526881] env[62736]: DEBUG oslo.service.loopingcall [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 585.527058] env[62736]: DEBUG nova.compute.manager [-] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 585.527144] env[62736]: DEBUG nova.network.neutron [-] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 585.530656] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 585.551520] env[62736]: DEBUG nova.network.neutron [-] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 585.551520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3ec439487f344477991dafcdac66082d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.558193] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3ec439487f344477991dafcdac66082d [ 585.582622] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 585.582912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 40ad55ddea154263bdc826b8b90b27a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.591302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40ad55ddea154263bdc826b8b90b27a8 [ 585.762235] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquiring lock "b88d76a9-6a30-4d5f-a5cd-f698e005dab5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.762473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "b88d76a9-6a30-4d5f-a5cd-f698e005dab5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.900729] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.479s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.901123] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 585.902914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg bf05613508dd4e71a67b0ee7618e7744 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.903910] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 22.582s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.905788] env[62736]: INFO nova.compute.claims [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 585.907232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 405743b861f743098abdcaabd9ab7373 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 585.949009] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405743b861f743098abdcaabd9ab7373 [ 585.951519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf05613508dd4e71a67b0ee7618e7744 [ 586.053501] env[62736]: DEBUG nova.network.neutron [-] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 586.053969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 192c7ce520a44530af91dd400f359acf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.063884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 192c7ce520a44530af91dd400f359acf [ 586.085140] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Releasing lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 586.085570] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 586.085759] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 586.086034] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2a56b035-d480-46e0-abf8-04f1a3a77245 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.096366] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03eb688f-ddc5-4ef0-b8c0-cf5a5d66a689 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.119527] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f537a3a9-b63a-4233-968a-942aca6ada0e could not be found. [ 586.119770] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 586.120025] env[62736]: INFO nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 586.120212] env[62736]: DEBUG oslo.service.loopingcall [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 586.120438] env[62736]: DEBUG nova.compute.manager [-] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 586.120524] env[62736]: DEBUG nova.network.neutron [-] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 586.134530] env[62736]: DEBUG nova.compute.manager [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Received event network-changed-3b1dae23-7093-47dd-9cbe-fa5054998b51 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 586.134712] env[62736]: DEBUG nova.compute.manager [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Refreshing instance network info cache due to event network-changed-3b1dae23-7093-47dd-9cbe-fa5054998b51. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 586.134956] env[62736]: DEBUG oslo_concurrency.lockutils [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] Acquiring lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 586.135256] env[62736]: DEBUG oslo_concurrency.lockutils [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] Acquired lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 586.135256] env[62736]: DEBUG nova.network.neutron [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Refreshing network info cache for port 3b1dae23-7093-47dd-9cbe-fa5054998b51 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 586.135941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] Expecting reply to msg ce4686f15e1d434abeeb11c820e06ffb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.137928] env[62736]: DEBUG nova.network.neutron [-] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 586.138346] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ce0a9a8a78d14972bb0c47249697a447 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.142415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce4686f15e1d434abeeb11c820e06ffb [ 586.147913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce0a9a8a78d14972bb0c47249697a447 [ 586.230452] env[62736]: DEBUG nova.compute.manager [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Received event network-vif-deleted-3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 586.230640] env[62736]: DEBUG nova.compute.manager [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Received event network-changed-e6cc30e4-2174-44e6-a3ca-ec741165ee60 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 586.230795] env[62736]: DEBUG nova.compute.manager [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Refreshing instance network info cache due to event network-changed-e6cc30e4-2174-44e6-a3ca-ec741165ee60. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 586.231311] env[62736]: DEBUG oslo_concurrency.lockutils [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] Acquiring lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 586.231416] env[62736]: DEBUG oslo_concurrency.lockutils [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] Acquired lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 586.231759] env[62736]: DEBUG nova.network.neutron [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Refreshing network info cache for port e6cc30e4-2174-44e6-a3ca-ec741165ee60 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 586.232254] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] Expecting reply to msg 0e7d8c8b23c84f7d93728d0d74fbb05f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.239338] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e7d8c8b23c84f7d93728d0d74fbb05f [ 586.410891] env[62736]: DEBUG nova.compute.utils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 586.411489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 46f42e475c6b480b9c9461f7edd75771 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.413408] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 232f3e319e6d403185300a8674936743 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.414298] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 586.414454] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 586.422597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 232f3e319e6d403185300a8674936743 [ 586.423081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46f42e475c6b480b9c9461f7edd75771 [ 586.485645] env[62736]: DEBUG nova.policy [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1d7d23e8e47c4620ba5a2d0de6d90280', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f778f88dc6e94351ad33c2d0cf875b3b', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 586.556377] env[62736]: INFO nova.compute.manager [-] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Took 1.03 seconds to deallocate network for instance. [ 586.558756] env[62736]: DEBUG nova.compute.claims [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 586.558950] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 586.618193] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "eeb7cdc7-427f-496f-be5d-60151a4bb0f3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 586.618420] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "eeb7cdc7-427f-496f-be5d-60151a4bb0f3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 586.642844] env[62736]: DEBUG nova.network.neutron [-] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 586.643092] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg defa0425c3894bee9ec12f08aa0897b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.651516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg defa0425c3894bee9ec12f08aa0897b0 [ 586.683878] env[62736]: DEBUG nova.network.neutron [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 586.794704] env[62736]: DEBUG nova.network.neutron [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 586.915541] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 586.917817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 0c56a89600ac4b0c8c9753ba6bb3af3b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.923335] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Successfully created port: 74c43113-48d4-426b-9dd2-405b9e6ab39f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 586.939793] env[62736]: DEBUG nova.network.neutron [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 586.941881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] Expecting reply to msg f777bc31fd754620b28f9fdafcf65d37 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.951073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f777bc31fd754620b28f9fdafcf65d37 [ 586.962166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c56a89600ac4b0c8c9753ba6bb3af3b [ 586.965277] env[62736]: DEBUG nova.network.neutron [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 586.966012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] Expecting reply to msg 468bc7cb9a8947bea5aa4cad82679618 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 586.976097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 468bc7cb9a8947bea5aa4cad82679618 [ 587.146255] env[62736]: INFO nova.compute.manager [-] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Took 1.03 seconds to deallocate network for instance. [ 587.147870] env[62736]: DEBUG nova.compute.claims [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 587.148066] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.358645] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ce670b8-f6b7-4d34-93c3-0e8e556e3e30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.368599] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0bda4147-f4ed-41fb-93f8-69b2e47cd619 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.405773] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12ab48e6-43ab-43e4-a823-18dfe649b858 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.413369] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-703bd237-1759-44f1-980c-c8b9c750555f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.429621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg c486f8eb2c7f484aa34ca42ac936cd49 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 587.430913] env[62736]: DEBUG nova.compute.provider_tree [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 587.436028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 71bf7ba1c6cc4d4aa88e2b5613e7f0f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 587.444520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71bf7ba1c6cc4d4aa88e2b5613e7f0f7 [ 587.447646] env[62736]: DEBUG oslo_concurrency.lockutils [req-55a88ce5-2aca-467d-b6c2-38b414b6eaeb req-c4667396-1b58-493c-932a-fed929712be4 service nova] Releasing lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 587.466573] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c486f8eb2c7f484aa34ca42ac936cd49 [ 587.470697] env[62736]: DEBUG oslo_concurrency.lockutils [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] Releasing lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 587.471016] env[62736]: DEBUG nova.compute.manager [req-5347357d-ff37-40ed-b41b-0d4285705952 req-2db3f039-f14e-4881-8c34-cfa88fda0682 service nova] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Received event network-vif-deleted-e6cc30e4-2174-44e6-a3ca-ec741165ee60 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 587.812491] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "1754ee22-ae95-4439-a6c0-200fd153bad0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.812754] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "1754ee22-ae95-4439-a6c0-200fd153bad0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.936808] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 587.940180] env[62736]: DEBUG nova.scheduler.client.report [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 587.942152] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg f903044e728f40f281ae31ac100a4cf7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 587.957380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f903044e728f40f281ae31ac100a4cf7 [ 587.960741] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 587.961140] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 587.961431] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 587.961764] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 587.962073] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 587.962338] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 587.962670] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 587.962951] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 587.963222] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 587.963483] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 587.963771] env[62736]: DEBUG nova.virt.hardware [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 587.965103] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8fdc086d-73c1-418c-9b9a-53931a23ab69 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 587.975746] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9c1530d-a23b-4ca1-9bee-8830f3242297 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.019704] env[62736]: ERROR nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 588.019704] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 588.019704] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 588.019704] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 588.019704] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 588.019704] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 588.019704] env[62736]: ERROR nova.compute.manager raise self.value [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 588.019704] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 588.019704] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 588.019704] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 588.020266] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 588.020266] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 588.020266] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 588.020266] env[62736]: ERROR nova.compute.manager [ 588.020266] env[62736]: Traceback (most recent call last): [ 588.020266] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 588.020266] env[62736]: listener.cb(fileno) [ 588.020266] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 588.020266] env[62736]: result = function(*args, **kwargs) [ 588.020266] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 588.020266] env[62736]: return func(*args, **kwargs) [ 588.020266] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 588.020266] env[62736]: raise e [ 588.020266] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 588.020266] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 588.020266] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 588.020266] env[62736]: created_port_ids = self._update_ports_for_instance( [ 588.020266] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 588.020266] env[62736]: with excutils.save_and_reraise_exception(): [ 588.020266] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 588.020266] env[62736]: self.force_reraise() [ 588.020266] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 588.020266] env[62736]: raise self.value [ 588.020266] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 588.020266] env[62736]: updated_port = self._update_port( [ 588.020266] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 588.020266] env[62736]: _ensure_no_port_binding_failure(port) [ 588.020266] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 588.020266] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 588.021247] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 588.021247] env[62736]: Removing descriptor: 16 [ 588.021596] env[62736]: ERROR nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Traceback (most recent call last): [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] yield resources [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.driver.spawn(context, instance, image_meta, [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] vm_ref = self.build_virtual_machine(instance, [ 588.021596] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] vif_infos = vmwarevif.get_vif_info(self._session, [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] for vif in network_info: [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self._sync_wrapper(fn, *args, **kwargs) [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.wait() [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self[:] = self._gt.wait() [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self._exit_event.wait() [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 588.021959] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] result = hub.switch() [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self.greenlet.switch() [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] result = function(*args, **kwargs) [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return func(*args, **kwargs) [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise e [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] nwinfo = self.network_api.allocate_for_instance( [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] created_port_ids = self._update_ports_for_instance( [ 588.022442] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] with excutils.save_and_reraise_exception(): [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.force_reraise() [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise self.value [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] updated_port = self._update_port( [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] _ensure_no_port_binding_failure(port) [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise exception.PortBindingFailed(port_id=port['id']) [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 588.022897] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] [ 588.023300] env[62736]: INFO nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Terminating instance [ 588.025843] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquiring lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 588.026088] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquired lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 588.026311] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 588.026817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 0c06e394dba24622b28ea99652b197b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.033618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c06e394dba24622b28ea99652b197b8 [ 588.445323] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.541s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 588.445905] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 588.447555] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 0e27ae842e304bb38aab0f2f97bc32f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.449176] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 23.311s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.450375] env[62736]: INFO nova.compute.claims [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 588.452100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 189846a5f7a54cfc9825e1f12bc90bd6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.499113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e27ae842e304bb38aab0f2f97bc32f7 [ 588.507946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 189846a5f7a54cfc9825e1f12bc90bd6 [ 588.548486] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 588.755394] env[62736]: DEBUG nova.compute.manager [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Received event network-vif-deleted-3b1dae23-7093-47dd-9cbe-fa5054998b51 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 588.755600] env[62736]: DEBUG nova.compute.manager [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Received event network-changed-74c43113-48d4-426b-9dd2-405b9e6ab39f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 588.755752] env[62736]: DEBUG nova.compute.manager [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Refreshing instance network info cache due to event network-changed-74c43113-48d4-426b-9dd2-405b9e6ab39f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 588.755928] env[62736]: DEBUG oslo_concurrency.lockutils [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] Acquiring lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 588.870434] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 588.870958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 2df14e7765344fe8aafd09c408b49b41 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.879625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2df14e7765344fe8aafd09c408b49b41 [ 588.955244] env[62736]: DEBUG nova.compute.utils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 588.955879] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 7fdb7c62d3e24f308bc511359a9769a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.957991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 174389e5e1804d64bc37f3ac26261a83 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 588.958931] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 588.959112] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 588.965740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 174389e5e1804d64bc37f3ac26261a83 [ 588.970278] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fdb7c62d3e24f308bc511359a9769a1 [ 589.018655] env[62736]: DEBUG nova.policy [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0df82cc3b5714a2389ac2a421876763f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '167ea2771f59412fbbb9ab0f2de63f68', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 589.237810] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "71ce0048-90fb-460c-a099-7eff5c092d35" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.237916] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "71ce0048-90fb-460c-a099-7eff5c092d35" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.313419] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Successfully created port: db4fac68-33e9-43e4-a979-d71a08eb0e06 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 589.373521] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Releasing lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 589.373960] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 589.374149] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 589.374452] env[62736]: DEBUG oslo_concurrency.lockutils [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] Acquired lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 589.374617] env[62736]: DEBUG nova.network.neutron [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Refreshing network info cache for port 74c43113-48d4-426b-9dd2-405b9e6ab39f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 589.375052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] Expecting reply to msg 9fe6a7db5775479b818276bbdf099c13 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 589.376303] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-778ddc4f-94a1-490e-8543-fbdf89902bd2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.382590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fe6a7db5775479b818276bbdf099c13 [ 589.387052] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ce89d21-e087-4a63-ae03-a7d3171b9b97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.417395] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance bd41759b-5c32-42e6-b625-4ea48f98045e could not be found. [ 589.417736] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 589.417980] env[62736]: INFO nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 589.418279] env[62736]: DEBUG oslo.service.loopingcall [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 589.418548] env[62736]: DEBUG nova.compute.manager [-] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 589.418696] env[62736]: DEBUG nova.network.neutron [-] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 589.460392] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 589.462215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 6ccdcf0a234b491aa32bae8eebf99b51 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 589.501526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ccdcf0a234b491aa32bae8eebf99b51 [ 589.530900] env[62736]: DEBUG nova.network.neutron [-] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 589.531364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 89bf5f8934284eb1a1da925a00a4fd1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 589.546206] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89bf5f8934284eb1a1da925a00a4fd1e [ 589.902721] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81b7db6f-0956-4ba9-a8ea-67e817ce7312 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.912042] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61f75ee2-91ba-4def-a5c2-c7ddb1d6442d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.947504] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77cabdc0-ffd9-4f0c-a934-1d35a28ae3dc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.956137] env[62736]: DEBUG nova.network.neutron [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 589.964277] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa697eb9-4b76-446c-b4be-8791cd3ed2a5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.970419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 8fa179619de64015beb312c880746b01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 589.980695] env[62736]: DEBUG nova.compute.provider_tree [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 589.981275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 6e06a33d9043477e8598670383cc6397 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.002417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e06a33d9043477e8598670383cc6397 [ 590.033680] env[62736]: DEBUG nova.network.neutron [-] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 590.035184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 50e5692c92d5416e8ba13ca71f507f90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.057441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fa179619de64015beb312c880746b01 [ 590.058008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50e5692c92d5416e8ba13ca71f507f90 [ 590.122045] env[62736]: DEBUG nova.network.neutron [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 590.122553] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] Expecting reply to msg d097c0054f1b40d18b259250972525c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.130728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d097c0054f1b40d18b259250972525c1 [ 590.285869] env[62736]: ERROR nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 590.285869] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.285869] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.285869] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.285869] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.285869] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.285869] env[62736]: ERROR nova.compute.manager raise self.value [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.285869] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 590.285869] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.285869] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 590.286471] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.286471] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 590.286471] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 590.286471] env[62736]: ERROR nova.compute.manager [ 590.286471] env[62736]: Traceback (most recent call last): [ 590.286471] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 590.286471] env[62736]: listener.cb(fileno) [ 590.286471] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.286471] env[62736]: result = function(*args, **kwargs) [ 590.286471] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 590.286471] env[62736]: return func(*args, **kwargs) [ 590.286471] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.286471] env[62736]: raise e [ 590.286471] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.286471] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 590.286471] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.286471] env[62736]: created_port_ids = self._update_ports_for_instance( [ 590.286471] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.286471] env[62736]: with excutils.save_and_reraise_exception(): [ 590.286471] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.286471] env[62736]: self.force_reraise() [ 590.286471] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.286471] env[62736]: raise self.value [ 590.286471] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.286471] env[62736]: updated_port = self._update_port( [ 590.286471] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.286471] env[62736]: _ensure_no_port_binding_failure(port) [ 590.286471] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.286471] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 590.287382] env[62736]: nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 590.287382] env[62736]: Removing descriptor: 16 [ 590.474220] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 590.484721] env[62736]: DEBUG nova.scheduler.client.report [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 590.487057] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 829a05999dba46cbad417c6eb7f09bec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.500036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 829a05999dba46cbad417c6eb7f09bec [ 590.506993] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 590.507295] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 590.507573] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 590.507767] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 590.507944] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 590.508281] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 590.508637] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 590.508849] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 590.509053] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 590.509248] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 590.509486] env[62736]: DEBUG nova.virt.hardware [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 590.510576] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebbe6a00-5dfc-4c6b-aef7-8c9bc7db746d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.518713] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f523c32-0cd3-4c18-9440-2f9e5956c863 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 590.534777] env[62736]: ERROR nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Traceback (most recent call last): [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] yield resources [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.driver.spawn(context, instance, image_meta, [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] vm_ref = self.build_virtual_machine(instance, [ 590.534777] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] vif_infos = vmwarevif.get_vif_info(self._session, [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] for vif in network_info: [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return self._sync_wrapper(fn, *args, **kwargs) [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.wait() [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self[:] = self._gt.wait() [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return self._exit_event.wait() [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 590.535168] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] current.throw(*self._exc) [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] result = function(*args, **kwargs) [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return func(*args, **kwargs) [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise e [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] nwinfo = self.network_api.allocate_for_instance( [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] created_port_ids = self._update_ports_for_instance( [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] with excutils.save_and_reraise_exception(): [ 590.535574] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.force_reraise() [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise self.value [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] updated_port = self._update_port( [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] _ensure_no_port_binding_failure(port) [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise exception.PortBindingFailed(port_id=port['id']) [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 590.535966] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] [ 590.535966] env[62736]: INFO nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Terminating instance [ 590.537366] env[62736]: INFO nova.compute.manager [-] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Took 1.12 seconds to deallocate network for instance. [ 590.537901] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquiring lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.538141] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquired lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 590.538289] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 590.538827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 3a54493518b44978a873c7c0f634f62e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.546542] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a54493518b44978a873c7c0f634f62e [ 590.549219] env[62736]: DEBUG nova.compute.claims [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 590.549466] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 590.567763] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 590.625930] env[62736]: DEBUG oslo_concurrency.lockutils [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] Releasing lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 590.626176] env[62736]: DEBUG nova.compute.manager [req-9fc60120-abdf-4dd8-b153-dbbf27ae1ef9 req-1000ddd1-c653-484c-af61-8e322a64f69b service nova] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Received event network-vif-deleted-74c43113-48d4-426b-9dd2-405b9e6ab39f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 590.668248] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 590.668248] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 82b732e8f3db43f39aa811c046739790 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.676087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82b732e8f3db43f39aa811c046739790 [ 590.987319] env[62736]: DEBUG nova.compute.manager [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Received event network-changed-db4fac68-33e9-43e4-a979-d71a08eb0e06 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 590.987595] env[62736]: DEBUG nova.compute.manager [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Refreshing instance network info cache due to event network-changed-db4fac68-33e9-43e4-a979-d71a08eb0e06. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 590.987728] env[62736]: DEBUG oslo_concurrency.lockutils [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] Acquiring lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.989783] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.541s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 590.990229] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 590.992018] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 4f24ac3d473b4fdebaa964abe1ad5dbd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 590.992978] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 21.058s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 590.993196] env[62736]: DEBUG nova.objects.instance [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lazy-loading 'resources' on Instance uuid 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91 {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 590.993504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 693aa18fce344cab95c9bbb6bf805445 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 591.001351] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 693aa18fce344cab95c9bbb6bf805445 [ 591.027091] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f24ac3d473b4fdebaa964abe1ad5dbd [ 591.170421] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Releasing lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 591.170421] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 591.170794] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 591.170908] env[62736]: DEBUG oslo_concurrency.lockutils [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] Acquired lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.171081] env[62736]: DEBUG nova.network.neutron [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Refreshing network info cache for port db4fac68-33e9-43e4-a979-d71a08eb0e06 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 591.171515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] Expecting reply to msg 439fbdef97ec455d8fb06333035ea0e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 591.172325] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9b79e291-1707-4f32-a009-68769c32dbe9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.181525] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4df03b78-d585-49a3-a5be-4dee9c5d769d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.191873] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 439fbdef97ec455d8fb06333035ea0e6 [ 591.203378] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 25508ff0-bc62-47b3-ae9e-7d9d8c735edc could not be found. [ 591.203602] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 591.203765] env[62736]: INFO nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Took 0.03 seconds to destroy the instance on the hypervisor. [ 591.204411] env[62736]: DEBUG oslo.service.loopingcall [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 591.204411] env[62736]: DEBUG nova.compute.manager [-] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 591.204411] env[62736]: DEBUG nova.network.neutron [-] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 591.222270] env[62736]: DEBUG nova.network.neutron [-] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 591.222797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 06969f70cac34c248ccaef0ba2ef3031 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 591.230573] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06969f70cac34c248ccaef0ba2ef3031 [ 591.365451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "20d9b891-9c2a-479c-8fd8-93405480dee4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.365667] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "20d9b891-9c2a-479c-8fd8-93405480dee4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.496875] env[62736]: DEBUG nova.compute.utils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 591.497506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 7a4f32547fee4ba0843c54527c165757 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 591.501166] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 591.501343] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 591.512618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a4f32547fee4ba0843c54527c165757 [ 591.582844] env[62736]: DEBUG nova.policy [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '16deda22cf4d47a6af9d8c18533df8cd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '761a5ab005a34ef2b788c40b495432d6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 591.713650] env[62736]: DEBUG nova.network.neutron [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 591.725402] env[62736]: DEBUG nova.network.neutron [-] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 591.725402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0d9f9eef25e64a64b79b2187aff690ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 591.735776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d9f9eef25e64a64b79b2187aff690ab [ 591.986452] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42cf0131-9f15-47ce-80dc-d5d65810d531 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.994321] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75f3d5ff-42a7-42a2-9b95-adfaf6e79761 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.027366] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 592.029336] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 8073ad427c0f4e55a4442d0d8863be07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 592.032781] env[62736]: DEBUG nova.network.neutron [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 592.033225] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] Expecting reply to msg 72c963605bff484bb3d8cba2245e9ead in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 592.034466] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bda53652-aa12-41c0-8543-7041394fa19c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.042124] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9f60d26-2fd6-403f-8a07-db47a7a76d32 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.046246] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72c963605bff484bb3d8cba2245e9ead [ 592.060728] env[62736]: DEBUG nova.compute.provider_tree [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.061233] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg b0aa79e127ae4c7490d4e54246a29188 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 592.065195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8073ad427c0f4e55a4442d0d8863be07 [ 592.069227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0aa79e127ae4c7490d4e54246a29188 [ 592.092957] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Successfully created port: d1d9c4d7-2cd5-43d7-8471-60c524b7cfda {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 592.228684] env[62736]: INFO nova.compute.manager [-] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Took 1.02 seconds to deallocate network for instance. [ 592.229796] env[62736]: DEBUG nova.compute.claims [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 592.229796] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.543041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 0efded45d1bf46c29690bcba7e702b08 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 592.543041] env[62736]: DEBUG oslo_concurrency.lockutils [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] Releasing lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.543041] env[62736]: DEBUG nova.compute.manager [req-071d9358-4044-4f6d-b5db-dddde452589e req-69a79fbe-17fe-4740-8bf5-c76f033c815c service nova] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Received event network-vif-deleted-db4fac68-33e9-43e4-a979-d71a08eb0e06 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 592.568828] env[62736]: DEBUG nova.scheduler.client.report [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 592.568828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg cf5f4f1a04644599b4d8147356763250 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 592.581829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0efded45d1bf46c29690bcba7e702b08 [ 592.587966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf5f4f1a04644599b4d8147356763250 [ 593.044530] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 593.085214] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 2.092s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 593.093059] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 593.093300] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 593.093502] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 593.093656] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 593.093821] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 593.093976] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 593.094176] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 593.094332] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 593.094490] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 593.094752] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 593.094800] env[62736]: DEBUG nova.virt.hardware [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 593.095492] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.466s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.097532] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 9959fe9959e44350991b77e9c46e1d4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 593.099176] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bac1996-a124-4ee1-a0fd-ee679f7d0f8d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.109649] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5052a286-5317-447c-bbb5-270197055a12 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.127064] env[62736]: INFO nova.scheduler.client.report [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Deleted allocations for instance 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91 [ 593.133764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 475a371984c8455195b893feb37aa8d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 593.143353] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquiring lock "48a0d4f8-4fab-454b-b065-e552647fba3f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 593.143758] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "48a0d4f8-4fab-454b-b065-e552647fba3f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 593.153798] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9959fe9959e44350991b77e9c46e1d4c [ 593.181906] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 475a371984c8455195b893feb37aa8d9 [ 593.497089] env[62736]: ERROR nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 593.497089] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 593.497089] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 593.497089] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 593.497089] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 593.497089] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 593.497089] env[62736]: ERROR nova.compute.manager raise self.value [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 593.497089] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 593.497089] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 593.497089] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 593.497679] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 593.497679] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 593.497679] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 593.497679] env[62736]: ERROR nova.compute.manager [ 593.497679] env[62736]: Traceback (most recent call last): [ 593.497679] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 593.497679] env[62736]: listener.cb(fileno) [ 593.497679] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 593.497679] env[62736]: result = function(*args, **kwargs) [ 593.497679] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 593.497679] env[62736]: return func(*args, **kwargs) [ 593.497679] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 593.497679] env[62736]: raise e [ 593.497679] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 593.497679] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 593.497679] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 593.497679] env[62736]: created_port_ids = self._update_ports_for_instance( [ 593.497679] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 593.497679] env[62736]: with excutils.save_and_reraise_exception(): [ 593.497679] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 593.497679] env[62736]: self.force_reraise() [ 593.497679] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 593.497679] env[62736]: raise self.value [ 593.497679] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 593.497679] env[62736]: updated_port = self._update_port( [ 593.497679] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 593.497679] env[62736]: _ensure_no_port_binding_failure(port) [ 593.497679] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 593.497679] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 593.498612] env[62736]: nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 593.498612] env[62736]: Removing descriptor: 16 [ 593.498612] env[62736]: ERROR nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Traceback (most recent call last): [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] yield resources [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.driver.spawn(context, instance, image_meta, [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 593.498612] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] vm_ref = self.build_virtual_machine(instance, [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] vif_infos = vmwarevif.get_vif_info(self._session, [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] for vif in network_info: [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self._sync_wrapper(fn, *args, **kwargs) [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.wait() [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self[:] = self._gt.wait() [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self._exit_event.wait() [ 593.499044] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] result = hub.switch() [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self.greenlet.switch() [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] result = function(*args, **kwargs) [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return func(*args, **kwargs) [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise e [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] nwinfo = self.network_api.allocate_for_instance( [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 593.499476] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] created_port_ids = self._update_ports_for_instance( [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] with excutils.save_and_reraise_exception(): [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.force_reraise() [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise self.value [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] updated_port = self._update_port( [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] _ensure_no_port_binding_failure(port) [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 593.499865] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise exception.PortBindingFailed(port_id=port['id']) [ 593.500257] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 593.500257] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] [ 593.500257] env[62736]: INFO nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Terminating instance [ 593.500385] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquiring lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 593.500484] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquired lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 593.500643] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 593.501041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg a6b92a9ffab7452ea24cf5d98a1b9970 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 593.513913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6b92a9ffab7452ea24cf5d98a1b9970 [ 593.585321] env[62736]: DEBUG nova.compute.manager [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Received event network-changed-d1d9c4d7-2cd5-43d7-8471-60c524b7cfda {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 593.585573] env[62736]: DEBUG nova.compute.manager [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Refreshing instance network info cache due to event network-changed-d1d9c4d7-2cd5-43d7-8471-60c524b7cfda. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 593.585784] env[62736]: DEBUG oslo_concurrency.lockutils [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] Acquiring lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 593.641042] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Lock "5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 27.692s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 593.641365] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6356ff88-32ef-4384-aee0-785ae5c97f0a tempest-ServerDiagnosticsV248Test-2096702815 tempest-ServerDiagnosticsV248Test-2096702815-project-member] Expecting reply to msg 7a5d2f5f40b746e4827133ad74e8b2d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 593.659900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a5d2f5f40b746e4827133ad74e8b2d7 [ 594.024196] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 594.031376] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ea7495b-313c-4320-a1e9-8a84d09bf0c3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.039762] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed0fc7fc-43bf-4f4d-a2ed-1ee6b8c4923d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.072616] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ff7cb89-d273-4a8e-86f6-4bb6bcb64670 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.080807] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ec113fd-8c32-43d0-8cdc-49efbe2c69cf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.098383] env[62736]: DEBUG nova.compute.provider_tree [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 594.098951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 9fc2714fb3a04c50a6ebb41e04cf2272 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 594.110883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fc2714fb3a04c50a6ebb41e04cf2272 [ 594.135174] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 594.135772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg c0a5ec69ffdc42b1ade51b90fa8e7c3e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 594.143779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0a5ec69ffdc42b1ade51b90fa8e7c3e [ 594.601972] env[62736]: DEBUG nova.scheduler.client.report [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 594.604350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 5b4255fe24f34a6cbf2f201808c12bde in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 594.619004] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b4255fe24f34a6cbf2f201808c12bde [ 594.638302] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Releasing lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 594.638733] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 594.638914] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 594.639214] env[62736]: DEBUG oslo_concurrency.lockutils [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] Acquired lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 594.639402] env[62736]: DEBUG nova.network.neutron [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Refreshing network info cache for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 594.639847] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] Expecting reply to msg ad669a0ea3cf46aabf52605dbdcd1fb0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 594.640768] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2a6fb39a-afc4-4f55-ba4d-7afbb347170a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.647309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad669a0ea3cf46aabf52605dbdcd1fb0 [ 594.650309] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e3e342d-661f-4991-bbd4-7a4bd105bb63 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 594.678399] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 918aa256-a10d-46d1-b9a4-ecd70cf3566f could not be found. [ 594.678765] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 594.678972] env[62736]: INFO nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 594.679354] env[62736]: DEBUG oslo.service.loopingcall [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 594.681075] env[62736]: DEBUG nova.compute.manager [-] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 594.681274] env[62736]: DEBUG nova.network.neutron [-] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 594.710186] env[62736]: DEBUG nova.network.neutron [-] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 594.710792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 066c1753f9ee44fba0f7dbda78a322e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 594.718202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 066c1753f9ee44fba0f7dbda78a322e0 [ 595.119383] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.016s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 595.119383] env[62736]: ERROR nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Traceback (most recent call last): [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.driver.spawn(context, instance, image_meta, [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self._vmops.spawn(context, instance, image_meta, injected_files, [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 595.119383] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] vm_ref = self.build_virtual_machine(instance, [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] vif_infos = vmwarevif.get_vif_info(self._session, [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] for vif in network_info: [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self._sync_wrapper(fn, *args, **kwargs) [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.wait() [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self[:] = self._gt.wait() [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self._exit_event.wait() [ 595.120032] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] result = hub.switch() [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return self.greenlet.switch() [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] result = function(*args, **kwargs) [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] return func(*args, **kwargs) [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise e [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] nwinfo = self.network_api.allocate_for_instance( [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 595.120497] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] created_port_ids = self._update_ports_for_instance( [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] with excutils.save_and_reraise_exception(): [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] self.force_reraise() [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise self.value [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] updated_port = self._update_port( [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] _ensure_no_port_binding_failure(port) [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 595.120890] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] raise exception.PortBindingFailed(port_id=port['id']) [ 595.121262] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] nova.exception.PortBindingFailed: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. [ 595.121262] env[62736]: ERROR nova.compute.manager [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] [ 595.121262] env[62736]: DEBUG nova.compute.utils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 595.121262] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.911s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 595.121262] env[62736]: INFO nova.compute.claims [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 595.121262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 093ab98859ec451e9a843efb4eece0d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.121949] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Build of instance 01b348db-6a0d-4283-b9d9-8edacfd1c219 was re-scheduled: Binding failed for port d3ab0149-beaf-4cc6-a12c-69e4bba83062, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 595.123828] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 595.124200] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquiring lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 595.124598] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Acquired lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 595.124985] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 595.125796] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 0951d7bbfeed49ff987428af9e81a2e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.132310] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0951d7bbfeed49ff987428af9e81a2e5 [ 595.162904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 093ab98859ec451e9a843efb4eece0d2 [ 595.164387] env[62736]: DEBUG nova.network.neutron [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 595.216329] env[62736]: DEBUG nova.network.neutron [-] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 595.216855] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e97ac441ce634611b4849c999dbf040f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.227964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e97ac441ce634611b4849c999dbf040f [ 595.311749] env[62736]: DEBUG nova.network.neutron [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 595.311749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] Expecting reply to msg efd87f1fbddb42769a925bea91905c74 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.328510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efd87f1fbddb42769a925bea91905c74 [ 595.628843] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 44fe46101f914b7385d65b0f5193aad8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.638018] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44fe46101f914b7385d65b0f5193aad8 [ 595.649143] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 595.671575] env[62736]: DEBUG nova.compute.manager [req-edac7500-aaeb-463e-8d8b-e2653e2c347c req-0258e7f8-1f8a-41fe-9038-5fe35aa333a2 service nova] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Received event network-vif-deleted-d1d9c4d7-2cd5-43d7-8471-60c524b7cfda {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 595.719198] env[62736]: INFO nova.compute.manager [-] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Took 1.04 seconds to deallocate network for instance. [ 595.721677] env[62736]: DEBUG nova.compute.claims [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 595.721865] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 595.801583] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 595.802116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg ae39be51488845d496921fe3c63d80a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 595.816370] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae39be51488845d496921fe3c63d80a9 [ 595.818877] env[62736]: DEBUG oslo_concurrency.lockutils [req-6da7853c-2607-46b8-aafa-16f47fb7fe02 req-6b29ebba-0ed7-4049-9dc4-a389d5b81cdc service nova] Releasing lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 596.304280] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Releasing lock "refresh_cache-01b348db-6a0d-4283-b9d9-8edacfd1c219" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 596.304547] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 596.304704] env[62736]: DEBUG nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 596.304867] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 596.320592] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 596.321156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg e24584e693644413a2fae07d8e707fc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 596.330663] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e24584e693644413a2fae07d8e707fc8 [ 596.570314] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ba0fac3-f8e0-4c25-9415-8d7f3967051d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.582972] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5562b2eb-e13b-4765-8410-e81e6110208f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.616808] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ef8785d-4952-41a9-be9c-b6ed49e9699e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.624733] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cc5b23f-4cad-4db3-a0a8-b41f19d42eb9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.637951] env[62736]: DEBUG nova.compute.provider_tree [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 596.639119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 6a9b4f8cbd0442bd8ccb53c4e10343f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 596.645657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a9b4f8cbd0442bd8ccb53c4e10343f2 [ 596.824649] env[62736]: DEBUG nova.network.neutron [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.825255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg adfc016323264ae08dc8da41032fead6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 596.837584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adfc016323264ae08dc8da41032fead6 [ 597.140906] env[62736]: DEBUG nova.scheduler.client.report [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 597.143859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 8b1826006780436ab9425e96f535039e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 597.157720] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b1826006780436ab9425e96f535039e [ 597.328051] env[62736]: INFO nova.compute.manager [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] [instance: 01b348db-6a0d-4283-b9d9-8edacfd1c219] Took 1.02 seconds to deallocate network for instance. [ 597.336586] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg b298200fd17344e78bfa8661cb62a4c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 597.370045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b298200fd17344e78bfa8661cb62a4c3 [ 597.645833] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.531s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 597.646367] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 597.648195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg f56ea046c86049ea8830d947cf36da0f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 597.649195] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.230s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 597.656125] env[62736]: INFO nova.compute.claims [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 597.656125] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 4177f6d1f6eb4274a8e2a2ad63278dd6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 597.685340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f56ea046c86049ea8830d947cf36da0f [ 597.693676] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4177f6d1f6eb4274a8e2a2ad63278dd6 [ 597.837534] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 26c5c4de2e504251aba3dcc2179e0fab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 597.873281] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26c5c4de2e504251aba3dcc2179e0fab [ 598.155567] env[62736]: DEBUG nova.compute.utils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 598.156251] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg f4c19715f16a4decac95f57a6476638b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 598.158285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg bb5dd76a0fe64248b7b5c41c670a76cb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 598.159049] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 598.159214] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 598.168876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb5dd76a0fe64248b7b5c41c670a76cb [ 598.170695] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4c19715f16a4decac95f57a6476638b [ 598.277553] env[62736]: DEBUG nova.policy [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd3df5db771204330838b73899c4646d0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f1f2d9e07c344faf9fd688396711d718', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 598.368771] env[62736]: INFO nova.scheduler.client.report [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Deleted allocations for instance 01b348db-6a0d-4283-b9d9-8edacfd1c219 [ 598.376120] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Expecting reply to msg 91132c3184ac4e77980b46c80833b51e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 598.389064] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91132c3184ac4e77980b46c80833b51e [ 598.663043] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 598.664959] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 8697439d843a4608af8adbf76b6fcb30 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 598.707119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8697439d843a4608af8adbf76b6fcb30 [ 598.877712] env[62736]: DEBUG oslo_concurrency.lockutils [None req-aa227588-b68c-441d-90f4-890883f45639 tempest-ListServerFiltersTestJSON-22237429 tempest-ListServerFiltersTestJSON-22237429-project-member] Lock "01b348db-6a0d-4283-b9d9-8edacfd1c219" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 66.592s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 598.878249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 6f381e57f69b4f49a0e8478d00f20a14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 598.892104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f381e57f69b4f49a0e8478d00f20a14 [ 599.051858] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Successfully created port: 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 599.103414] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74e5abd0-823d-4dca-9539-cd1359ae67ef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.111669] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89f3acaa-251a-41e9-a318-301d11595df3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.156400] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-512ffc12-07c3-42db-82e6-8046259be7f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.164271] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c36303fb-d669-4d1a-979f-03c92512d5a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.177699] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg d3efb99a1c7f42ffabeaf109d50fe1df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 599.188809] env[62736]: DEBUG nova.compute.provider_tree [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 599.189462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 201bd216f70043b798b5cc44ab39e7bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 599.196853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 201bd216f70043b798b5cc44ab39e7bb [ 599.220477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3efb99a1c7f42ffabeaf109d50fe1df [ 599.380236] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 599.382037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d9e0e1026dd340c58803da19c246267c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 599.429174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9e0e1026dd340c58803da19c246267c [ 599.681253] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 599.697995] env[62736]: DEBUG nova.scheduler.client.report [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 599.700559] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 4f41c1e07b4f40d2a50e239d8cc1580d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 599.712681] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:31:10Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='1329964930',id=22,is_public=True,memory_mb=192,name='tempest-flavor_with_ephemeral_0-107728853',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 599.712972] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 599.713157] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 599.713345] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 599.713489] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 599.713630] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 599.713826] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 599.713977] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 599.714134] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 599.714288] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 599.714463] env[62736]: DEBUG nova.virt.hardware [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 599.715550] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0db901e9-d2eb-4760-851c-4ada8685a927 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.718528] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f41c1e07b4f40d2a50e239d8cc1580d [ 599.724939] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fb4ac0c-6e8e-4036-9944-6a13c22e62fa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.903704] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.208027] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.556s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 600.208027] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 600.208027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg f3f7b5a6f2ce4c6d90f484df1c353c42 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 600.209477] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.350s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.215400] env[62736]: INFO nova.compute.claims [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 600.217174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 0eab55873d2647f69daa00528ce228fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 600.243069] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3f7b5a6f2ce4c6d90f484df1c353c42 [ 600.268280] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0eab55873d2647f69daa00528ce228fe [ 600.323865] env[62736]: ERROR nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 600.323865] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 600.323865] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 600.323865] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 600.323865] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 600.323865] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 600.323865] env[62736]: ERROR nova.compute.manager raise self.value [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 600.323865] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 600.323865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 600.323865] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 600.324389] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 600.324389] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 600.324389] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 600.324389] env[62736]: ERROR nova.compute.manager [ 600.324389] env[62736]: Traceback (most recent call last): [ 600.324389] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 600.324389] env[62736]: listener.cb(fileno) [ 600.324389] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 600.324389] env[62736]: result = function(*args, **kwargs) [ 600.324389] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 600.324389] env[62736]: return func(*args, **kwargs) [ 600.324389] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 600.324389] env[62736]: raise e [ 600.324389] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 600.324389] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 600.324389] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 600.324389] env[62736]: created_port_ids = self._update_ports_for_instance( [ 600.324389] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 600.324389] env[62736]: with excutils.save_and_reraise_exception(): [ 600.324389] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 600.324389] env[62736]: self.force_reraise() [ 600.324389] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 600.324389] env[62736]: raise self.value [ 600.324389] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 600.324389] env[62736]: updated_port = self._update_port( [ 600.324389] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 600.324389] env[62736]: _ensure_no_port_binding_failure(port) [ 600.324389] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 600.324389] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 600.325201] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 600.325201] env[62736]: Removing descriptor: 16 [ 600.325201] env[62736]: ERROR nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Traceback (most recent call last): [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] yield resources [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.driver.spawn(context, instance, image_meta, [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self._vmops.spawn(context, instance, image_meta, injected_files, [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 600.325201] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] vm_ref = self.build_virtual_machine(instance, [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] vif_infos = vmwarevif.get_vif_info(self._session, [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] for vif in network_info: [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self._sync_wrapper(fn, *args, **kwargs) [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.wait() [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self[:] = self._gt.wait() [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self._exit_event.wait() [ 600.325560] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] result = hub.switch() [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self.greenlet.switch() [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] result = function(*args, **kwargs) [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return func(*args, **kwargs) [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise e [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] nwinfo = self.network_api.allocate_for_instance( [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 600.325950] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] created_port_ids = self._update_ports_for_instance( [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] with excutils.save_and_reraise_exception(): [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.force_reraise() [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise self.value [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] updated_port = self._update_port( [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] _ensure_no_port_binding_failure(port) [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 600.326335] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise exception.PortBindingFailed(port_id=port['id']) [ 600.327067] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 600.327067] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] [ 600.327067] env[62736]: INFO nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Terminating instance [ 600.327176] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquiring lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.327274] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquired lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.327441] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 600.327853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 68c2ae68dfe842c1b7954c93c4603e24 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 600.334299] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68c2ae68dfe842c1b7954c93c4603e24 [ 600.532567] env[62736]: DEBUG nova.compute.manager [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Received event network-changed-2fc1b9dc-5720-4a4e-8ecd-4e25c684be47 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 600.532830] env[62736]: DEBUG nova.compute.manager [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Refreshing instance network info cache due to event network-changed-2fc1b9dc-5720-4a4e-8ecd-4e25c684be47. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 600.533189] env[62736]: DEBUG oslo_concurrency.lockutils [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] Acquiring lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.720606] env[62736]: DEBUG nova.compute.utils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 600.721262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg f929c4cc8f1440acba2661dd6af9cda5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 600.723637] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 87dc4eb71372458cbd3e41974a94d188 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 600.724879] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 600.725042] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 600.730348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87dc4eb71372458cbd3e41974a94d188 [ 600.735165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f929c4cc8f1440acba2661dd6af9cda5 [ 600.777324] env[62736]: DEBUG nova.policy [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '283bddf95dd940438dacbce6ef1d3733', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'bc21137f57474824a35a5f0926cc04ef', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 600.852691] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.020389] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 601.021180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 82b7eefc64be4b5c9087ded96ae7c0db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.029218] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82b7eefc64be4b5c9087ded96ae7c0db [ 601.133990] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Successfully created port: ad867a88-c3d1-405e-be2f-5e3328b9c9b9 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 601.226122] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 601.227877] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 8bf80850032b410d975dbdc7f5391945 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.288632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8bf80850032b410d975dbdc7f5391945 [ 601.525369] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Releasing lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 601.525787] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 601.526079] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 601.527857] env[62736]: DEBUG oslo_concurrency.lockutils [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] Acquired lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.527857] env[62736]: DEBUG nova.network.neutron [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Refreshing network info cache for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 601.527857] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] Expecting reply to msg b5364cdf8e8749f3b0546ca215262ae2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.528173] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-45313908-71d7-486d-b42a-31869fc5c059 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.538071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "459055aa-ae56-480f-b2c6-e45cc05bb50c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.538071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "459055aa-ae56-480f-b2c6-e45cc05bb50c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.540332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5364cdf8e8749f3b0546ca215262ae2 [ 601.543962] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6341132-93a5-4342-87d2-53224f22eb56 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.579528] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 64ee370c-6d43-4e2b-a27a-3aa716fdc322 could not be found. [ 601.579776] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 601.579955] env[62736]: INFO nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Took 0.05 seconds to destroy the instance on the hypervisor. [ 601.580238] env[62736]: DEBUG oslo.service.loopingcall [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 601.581546] env[62736]: DEBUG nova.compute.manager [-] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 601.581546] env[62736]: DEBUG nova.network.neutron [-] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 601.600369] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "5041b2c1-e7f1-4039-b7c4-1f677918025a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.600611] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "5041b2c1-e7f1-4039-b7c4-1f677918025a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.633910] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "896c2790-9107-4d1c-811b-07a87582c0a1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.634471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "896c2790-9107-4d1c-811b-07a87582c0a1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.635731] env[62736]: DEBUG nova.network.neutron [-] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 601.636411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 49023230d14f44b0afc336e4046bbb4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.647026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49023230d14f44b0afc336e4046bbb4c [ 601.743578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg f7df6fe6818740eaa057315fe5f40235 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.777055] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4aee3e47-9fbf-41b0-b431-0d8c5db732b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.788049] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4de8a497-c967-4e41-939d-e791727337b0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.821663] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7df6fe6818740eaa057315fe5f40235 [ 601.822702] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7000b2c3-ae6c-481c-b7e7-ce9aa9d2ba5d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.831455] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a00f265b-4b9c-474b-a15e-821d8267f2c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.847337] env[62736]: DEBUG nova.compute.provider_tree [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 601.848281] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 37b957e4f1ae40ccbabcb76397b94cab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 601.856548] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37b957e4f1ae40ccbabcb76397b94cab [ 601.932769] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquiring lock "1174dcea-6ba6-4a2b-9eb4-85e662691c30" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.933740] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "1174dcea-6ba6-4a2b-9eb4-85e662691c30" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.055511] env[62736]: DEBUG nova.network.neutron [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 602.143277] env[62736]: DEBUG nova.network.neutron [-] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.143759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c3a77ce4fc814aa99741ef19cd97592c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.154404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3a77ce4fc814aa99741ef19cd97592c [ 602.218394] env[62736]: DEBUG nova.network.neutron [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.218931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] Expecting reply to msg 5899d8435c9a4cfb82cbbf0db6933393 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.226399] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5899d8435c9a4cfb82cbbf0db6933393 [ 602.248039] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 602.276183] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 602.278910] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 602.278910] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 602.278910] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 602.278910] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 602.279350] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 602.279752] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 602.280154] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 602.280530] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 602.280852] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 602.281429] env[62736]: DEBUG nova.virt.hardware [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 602.282467] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e1a14ce-1917-4457-9033-a783ca34582c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.293082] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a394846-0981-498d-90c2-f63826cba7cf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 602.350327] env[62736]: DEBUG nova.scheduler.client.report [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 602.352784] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 3600a368cbd648d8ab34b0dbc29325e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.364665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3600a368cbd648d8ab34b0dbc29325e7 [ 602.491933] env[62736]: ERROR nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 602.491933] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 602.491933] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 602.491933] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 602.491933] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 602.491933] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 602.491933] env[62736]: ERROR nova.compute.manager raise self.value [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 602.491933] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 602.491933] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 602.491933] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 602.492552] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 602.492552] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 602.492552] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 602.492552] env[62736]: ERROR nova.compute.manager [ 602.492552] env[62736]: Traceback (most recent call last): [ 602.492552] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 602.492552] env[62736]: listener.cb(fileno) [ 602.492552] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 602.492552] env[62736]: result = function(*args, **kwargs) [ 602.492552] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 602.492552] env[62736]: return func(*args, **kwargs) [ 602.492552] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 602.492552] env[62736]: raise e [ 602.492552] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 602.492552] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 602.492552] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 602.492552] env[62736]: created_port_ids = self._update_ports_for_instance( [ 602.492552] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 602.492552] env[62736]: with excutils.save_and_reraise_exception(): [ 602.492552] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 602.492552] env[62736]: self.force_reraise() [ 602.492552] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 602.492552] env[62736]: raise self.value [ 602.492552] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 602.492552] env[62736]: updated_port = self._update_port( [ 602.492552] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 602.492552] env[62736]: _ensure_no_port_binding_failure(port) [ 602.492552] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 602.492552] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 602.493510] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 602.493510] env[62736]: Removing descriptor: 16 [ 602.493510] env[62736]: ERROR nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] Traceback (most recent call last): [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] yield resources [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.driver.spawn(context, instance, image_meta, [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 602.493510] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] vm_ref = self.build_virtual_machine(instance, [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] for vif in network_info: [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self._sync_wrapper(fn, *args, **kwargs) [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.wait() [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self[:] = self._gt.wait() [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self._exit_event.wait() [ 602.494163] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] result = hub.switch() [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self.greenlet.switch() [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] result = function(*args, **kwargs) [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return func(*args, **kwargs) [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise e [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] nwinfo = self.network_api.allocate_for_instance( [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 602.494620] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] created_port_ids = self._update_ports_for_instance( [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] with excutils.save_and_reraise_exception(): [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.force_reraise() [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise self.value [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] updated_port = self._update_port( [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] _ensure_no_port_binding_failure(port) [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 602.495053] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise exception.PortBindingFailed(port_id=port['id']) [ 602.495460] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 602.495460] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] [ 602.495460] env[62736]: INFO nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Terminating instance [ 602.496748] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquiring lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.496900] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquired lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 602.497061] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 602.497496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg a25ec583ba274a77808b9cf09188a908 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.503657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a25ec583ba274a77808b9cf09188a908 [ 602.646241] env[62736]: INFO nova.compute.manager [-] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Took 1.07 seconds to deallocate network for instance. [ 602.648995] env[62736]: DEBUG nova.compute.claims [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 602.649179] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.721042] env[62736]: DEBUG oslo_concurrency.lockutils [req-3078dc6b-bceb-4378-8c41-a9f0c9e04311 req-5df28410-1f2a-4ad1-aa91-4423d6cc1088 service nova] Releasing lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.784146] env[62736]: DEBUG nova.compute.manager [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Received event network-vif-deleted-2fc1b9dc-5720-4a4e-8ecd-4e25c684be47 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 602.784360] env[62736]: DEBUG nova.compute.manager [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Received event network-changed-ad867a88-c3d1-405e-be2f-5e3328b9c9b9 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 602.784530] env[62736]: DEBUG nova.compute.manager [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Refreshing instance network info cache due to event network-changed-ad867a88-c3d1-405e-be2f-5e3328b9c9b9. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 602.784719] env[62736]: DEBUG oslo_concurrency.lockutils [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] Acquiring lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.855866] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.647s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.856693] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 602.858265] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg e3484df571d64e9c92bb3d58bb1ca9e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.859331] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.146s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.861061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg e89ac885c21648c49b065b20a4f3e671 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 602.903995] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3484df571d64e9c92bb3d58bb1ca9e3 [ 602.908985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e89ac885c21648c49b065b20a4f3e671 [ 603.018240] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 603.108188] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 603.108732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 15088a0b5bff47ad93c624b90cac060f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.117418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15088a0b5bff47ad93c624b90cac060f [ 603.364108] env[62736]: DEBUG nova.compute.utils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 603.364741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg c311ddf2dff548fa85156ec54c4a137f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.365643] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 603.365811] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 603.378414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c311ddf2dff548fa85156ec54c4a137f [ 603.438757] env[62736]: DEBUG nova.policy [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4f17ce167bc14a11977fcecdf1420e7d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c4c8d1a186584c33a1f71c1239b6d670', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 603.612311] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Releasing lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 603.612733] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 603.612909] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 603.613458] env[62736]: DEBUG oslo_concurrency.lockutils [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] Acquired lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 603.613562] env[62736]: DEBUG nova.network.neutron [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Refreshing network info cache for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 603.614006] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] Expecting reply to msg c3f85929d5f4464e966796c88d210945 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.614737] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b145aed9-2c05-4d02-8036-750c5759ba2f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.622809] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3f85929d5f4464e966796c88d210945 [ 603.628563] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5412bc53-c6e3-4925-816a-6e8baa95af8d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.664441] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3a848010-382b-4753-8420-37222963e7e2 could not be found. [ 603.664751] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 603.664883] env[62736]: INFO nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Took 0.05 seconds to destroy the instance on the hypervisor. [ 603.665138] env[62736]: DEBUG oslo.service.loopingcall [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 603.667656] env[62736]: DEBUG nova.compute.manager [-] [instance: 3a848010-382b-4753-8420-37222963e7e2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 603.667752] env[62736]: DEBUG nova.network.neutron [-] [instance: 3a848010-382b-4753-8420-37222963e7e2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 603.689263] env[62736]: DEBUG nova.network.neutron [-] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 603.689931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c7b6d3fc1c0043c9852b44c113a3813c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.696831] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7b6d3fc1c0043c9852b44c113a3813c [ 603.833633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-510cfd54-9db6-47c3-8364-46c37d089528 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.841467] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-710bd9b6-bbeb-4bd1-9433-fa548bfb2402 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.888322] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 603.888322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg ce40318b49904ddca5273d3b20a48e9d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.888322] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8dd6703-3450-44aa-b976-57d2b20f94df {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.889890] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eccc071f-f5db-4a2f-8339-4f7b04fbb505 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 603.905278] env[62736]: DEBUG nova.compute.provider_tree [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 603.905774] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg b9d1b2a7ec6a495daea7cb3fbfa15ec4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 603.913786] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9d1b2a7ec6a495daea7cb3fbfa15ec4 [ 603.925008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce40318b49904ddca5273d3b20a48e9d [ 604.024353] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Successfully created port: 4a3a996d-9a97-4d00-a63e-71097cd065b2 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 604.136688] env[62736]: DEBUG nova.network.neutron [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 604.191938] env[62736]: DEBUG nova.network.neutron [-] [instance: 3a848010-382b-4753-8420-37222963e7e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.192842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 03ba29d0ffba45ef8bab4b746388087e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.201778] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03ba29d0ffba45ef8bab4b746388087e [ 604.303895] env[62736]: DEBUG nova.network.neutron [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.304612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] Expecting reply to msg 46fca9e3ceaf47c58014d202fcaa746e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.313015] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46fca9e3ceaf47c58014d202fcaa746e [ 604.385902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg f42871096ad24840a60603bbf5f41875 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.411560] env[62736]: DEBUG nova.scheduler.client.report [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 604.411560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 3a846c6991a14205a01fc20392fe3a6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.430163] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a846c6991a14205a01fc20392fe3a6c [ 604.430986] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f42871096ad24840a60603bbf5f41875 [ 604.697966] env[62736]: INFO nova.compute.manager [-] [instance: 3a848010-382b-4753-8420-37222963e7e2] Took 1.03 seconds to deallocate network for instance. [ 604.700788] env[62736]: DEBUG nova.compute.claims [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 604.700788] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.808038] env[62736]: DEBUG oslo_concurrency.lockutils [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] Releasing lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 604.808038] env[62736]: DEBUG nova.compute.manager [req-497724d7-abe4-46a8-8d03-7e6fab9cee6d req-f11872d8-c8bc-4546-874c-920a8033020c service nova] [instance: 3a848010-382b-4753-8420-37222963e7e2] Received event network-vif-deleted-ad867a88-c3d1-405e-be2f-5e3328b9c9b9 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 604.890187] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 604.918410] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 604.918410] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 604.918410] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 604.918561] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 604.918561] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 604.918561] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 604.918561] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 604.918561] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 604.918740] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 604.918740] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 604.918740] env[62736]: DEBUG nova.virt.hardware [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 604.918740] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.057s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.918740] env[62736]: ERROR nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Traceback (most recent call last): [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.driver.spawn(context, instance, image_meta, [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] vm_ref = self.build_virtual_machine(instance, [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] vif_infos = vmwarevif.get_vif_info(self._session, [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] for vif in network_info: [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 604.918908] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self._sync_wrapper(fn, *args, **kwargs) [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.wait() [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self[:] = self._gt.wait() [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self._exit_event.wait() [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] result = hub.switch() [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return self.greenlet.switch() [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] result = function(*args, **kwargs) [ 604.919324] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] return func(*args, **kwargs) [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise e [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] nwinfo = self.network_api.allocate_for_instance( [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] created_port_ids = self._update_ports_for_instance( [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] with excutils.save_and_reraise_exception(): [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] self.force_reraise() [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 604.919729] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise self.value [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] updated_port = self._update_port( [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] _ensure_no_port_binding_failure(port) [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] raise exception.PortBindingFailed(port_id=port['id']) [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] nova.exception.PortBindingFailed: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. [ 604.920132] env[62736]: ERROR nova.compute.manager [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] [ 604.920132] env[62736]: DEBUG nova.compute.utils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 604.921462] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe8c7fc6-cac1-4cb3-a9ac-2ea6fa9adfbd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.923820] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.365s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.926113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 800bfc676e864be39fb6fb71da635a17 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.928259] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Build of instance 958e85c0-cb75-4dfd-92c4-b68957e87ec5 was re-scheduled: Binding failed for port 3fe70ba4-a5cc-4008-a5e7-05d2f9a3ea50, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 604.928821] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 604.929050] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquiring lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 604.929194] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Acquired lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 604.929377] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 604.929746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg c19984e4ceb44b66ba89668348a87099 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.936673] env[62736]: DEBUG nova.compute.manager [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Received event network-changed-4a3a996d-9a97-4d00-a63e-71097cd065b2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 604.936673] env[62736]: DEBUG nova.compute.manager [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Refreshing instance network info cache due to event network-changed-4a3a996d-9a97-4d00-a63e-71097cd065b2. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 604.936673] env[62736]: DEBUG oslo_concurrency.lockutils [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] Acquiring lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 604.936673] env[62736]: DEBUG oslo_concurrency.lockutils [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] Acquired lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 604.936673] env[62736]: DEBUG nova.network.neutron [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Refreshing network info cache for port 4a3a996d-9a97-4d00-a63e-71097cd065b2 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 604.937065] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] Expecting reply to msg 2c9a478cffc14e87a198fae0be8d95fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 604.938995] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-203bdf37-a8ea-4f44-91a1-7f395df19fc7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.944662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c19984e4ceb44b66ba89668348a87099 [ 604.945275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c9a478cffc14e87a198fae0be8d95fa [ 604.986480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 800bfc676e864be39fb6fb71da635a17 [ 605.067935] env[62736]: ERROR nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 605.067935] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 605.067935] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 605.067935] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 605.067935] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 605.067935] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 605.067935] env[62736]: ERROR nova.compute.manager raise self.value [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 605.067935] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 605.067935] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 605.067935] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 605.068512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 605.068512] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 605.068512] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 605.068512] env[62736]: ERROR nova.compute.manager [ 605.068512] env[62736]: Traceback (most recent call last): [ 605.068512] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 605.068512] env[62736]: listener.cb(fileno) [ 605.068512] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 605.068512] env[62736]: result = function(*args, **kwargs) [ 605.068512] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 605.068512] env[62736]: return func(*args, **kwargs) [ 605.068512] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 605.068512] env[62736]: raise e [ 605.068512] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 605.068512] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 605.068512] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 605.068512] env[62736]: created_port_ids = self._update_ports_for_instance( [ 605.068512] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 605.068512] env[62736]: with excutils.save_and_reraise_exception(): [ 605.068512] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 605.068512] env[62736]: self.force_reraise() [ 605.068512] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 605.068512] env[62736]: raise self.value [ 605.068512] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 605.068512] env[62736]: updated_port = self._update_port( [ 605.068512] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 605.068512] env[62736]: _ensure_no_port_binding_failure(port) [ 605.068512] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 605.068512] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 605.069517] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 605.069517] env[62736]: Removing descriptor: 16 [ 605.069517] env[62736]: ERROR nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Traceback (most recent call last): [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] yield resources [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.driver.spawn(context, instance, image_meta, [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 605.069517] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] vm_ref = self.build_virtual_machine(instance, [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] vif_infos = vmwarevif.get_vif_info(self._session, [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] for vif in network_info: [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self._sync_wrapper(fn, *args, **kwargs) [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.wait() [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self[:] = self._gt.wait() [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self._exit_event.wait() [ 605.069917] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] result = hub.switch() [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self.greenlet.switch() [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] result = function(*args, **kwargs) [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return func(*args, **kwargs) [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise e [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] nwinfo = self.network_api.allocate_for_instance( [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 605.070370] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] created_port_ids = self._update_ports_for_instance( [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] with excutils.save_and_reraise_exception(): [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.force_reraise() [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise self.value [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] updated_port = self._update_port( [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] _ensure_no_port_binding_failure(port) [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 605.070982] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise exception.PortBindingFailed(port_id=port['id']) [ 605.071437] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 605.071437] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] [ 605.071437] env[62736]: INFO nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Terminating instance [ 605.071547] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquiring lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 605.452392] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 605.469072] env[62736]: DEBUG nova.network.neutron [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 605.553927] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.554437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 2150671b99e646878352b6e3eb04c191 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 605.565861] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2150671b99e646878352b6e3eb04c191 [ 605.654129] env[62736]: DEBUG nova.network.neutron [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 605.654638] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] Expecting reply to msg 043ae222bebb4ba4a8e416a34dad3ed5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 605.664335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 043ae222bebb4ba4a8e416a34dad3ed5 [ 605.921140] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e590255-9323-4dec-a6e7-5a36d28ed32e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.928944] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9a552cf-f770-4cb8-b478-96455854b843 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.960184] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a47b0895-2f35-47ab-9dd2-8b94e17ccd21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.967789] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9456b322-cf63-4069-87bd-879140aa5ca1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.981964] env[62736]: DEBUG nova.compute.provider_tree [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 605.982396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 9d9d81ca0f494a61913308e016b19e5d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 605.994723] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d9d81ca0f494a61913308e016b19e5d [ 606.058858] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Releasing lock "refresh_cache-958e85c0-cb75-4dfd-92c4-b68957e87ec5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 606.059106] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 606.059546] env[62736]: DEBUG nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 606.059725] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 606.084688] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 606.085260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg d767de38805c48ab87d1af95fb2d778c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 606.092639] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d767de38805c48ab87d1af95fb2d778c [ 606.156596] env[62736]: DEBUG oslo_concurrency.lockutils [req-505bd245-105c-427d-984e-e968437c7953 req-ca0561a6-58d0-4ab6-8e9b-76f67923ae0d service nova] Releasing lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 606.157015] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquired lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 606.157250] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 606.157686] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg ccda1e8379ec4859879c6883cd938c67 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 606.165032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccda1e8379ec4859879c6883cd938c67 [ 606.489802] env[62736]: DEBUG nova.scheduler.client.report [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 606.492279] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 41fdec7b0a7d4d949290f6e0642af82d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 606.508249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41fdec7b0a7d4d949290f6e0642af82d [ 606.587906] env[62736]: DEBUG nova.network.neutron [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 606.588288] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 5ff88dbe4f724a7084fcb64a6226d5ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 606.602777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ff88dbe4f724a7084fcb64a6226d5ab [ 606.679314] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 606.918499] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 606.918961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 76eeeaa5b2ac424196acc90a97ffd5de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 606.928667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76eeeaa5b2ac424196acc90a97ffd5de [ 606.994751] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.071s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.995359] env[62736]: ERROR nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Traceback (most recent call last): [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.driver.spawn(context, instance, image_meta, [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] vm_ref = self.build_virtual_machine(instance, [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] vif_infos = vmwarevif.get_vif_info(self._session, [ 606.995359] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] for vif in network_info: [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self._sync_wrapper(fn, *args, **kwargs) [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.wait() [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self[:] = self._gt.wait() [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self._exit_event.wait() [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] result = hub.switch() [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 606.995746] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return self.greenlet.switch() [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] result = function(*args, **kwargs) [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] return func(*args, **kwargs) [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise e [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] nwinfo = self.network_api.allocate_for_instance( [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] created_port_ids = self._update_ports_for_instance( [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] with excutils.save_and_reraise_exception(): [ 606.996188] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] self.force_reraise() [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise self.value [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] updated_port = self._update_port( [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] _ensure_no_port_binding_failure(port) [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] raise exception.PortBindingFailed(port_id=port['id']) [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] nova.exception.PortBindingFailed: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. [ 606.996573] env[62736]: ERROR nova.compute.manager [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] [ 606.996915] env[62736]: DEBUG nova.compute.utils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 606.998862] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.851s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 607.002275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 215b3c4e44fc4afabab5773df4cd66a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.003720] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Build of instance 52f8e827-3faf-47c1-b793-ccacc5a0bc4c was re-scheduled: Binding failed for port e6cc30e4-2174-44e6-a3ca-ec741165ee60, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 607.004129] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 607.004360] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquiring lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 607.004507] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Acquired lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 607.004661] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 607.005027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 30c538abe4f842ba82ff978baceea525 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.018999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30c538abe4f842ba82ff978baceea525 [ 607.039758] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 215b3c4e44fc4afabab5773df4cd66a0 [ 607.095688] env[62736]: INFO nova.compute.manager [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] [instance: 958e85c0-cb75-4dfd-92c4-b68957e87ec5] Took 1.04 seconds to deallocate network for instance. [ 607.096633] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 25e4b55eeba04df5963cdd3ec601d6aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.100517] env[62736]: DEBUG nova.compute.manager [req-73451db3-ddc0-43b1-b0de-1829e330fc94 req-aea619c2-50f7-45ce-b05c-c0ecdc6248bf service nova] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Received event network-vif-deleted-4a3a996d-9a97-4d00-a63e-71097cd065b2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 607.144379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25e4b55eeba04df5963cdd3ec601d6aa [ 607.628100] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Releasing lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 607.628100] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 607.628100] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 607.628100] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cb386a75-7530-44f5-a085-387bab7f0175 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.628100] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb11091a-876f-449c-bcf2-37f65377aa79 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 607.628226] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c7670280-b5a7-4de2-8cf9-6814a81eb1a6 could not be found. [ 607.628226] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 607.628226] env[62736]: INFO nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 607.628226] env[62736]: DEBUG oslo.service.loopingcall [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 607.628226] env[62736]: DEBUG nova.compute.manager [-] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 607.628226] env[62736]: DEBUG nova.network.neutron [-] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 607.628465] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg ebde4b7cb77248049687b81e32cc1e03 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.645716] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 607.656966] env[62736]: DEBUG nova.network.neutron [-] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 607.657536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bec33a204c5748e3b9d61d06480c08a2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.691897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bec33a204c5748e3b9d61d06480c08a2 [ 607.700642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebde4b7cb77248049687b81e32cc1e03 [ 607.714652] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquiring lock "1a5a4c2e-74e6-4a62-adda-c3d89deecf05" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 607.714890] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "1a5a4c2e-74e6-4a62-adda-c3d89deecf05" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 607.746446] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 607.746973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 7694eea51b944c51afeb2f4a21452514 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 607.758642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7694eea51b944c51afeb2f4a21452514 [ 608.058257] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afecf385-5b0a-47d1-993c-ffd73baec100 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.066074] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ddefbe8-a022-44fe-af29-b3690eca8f91 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.096792] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a10246d-5c69-402c-a44e-9abd1642046f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.103938] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-948bb5e3-950e-40b5-84a0-7a237abf072a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 608.120311] env[62736]: DEBUG nova.compute.provider_tree [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 608.120955] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 891cf6615d2b43d4a894db1889d9dd01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.129182] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 891cf6615d2b43d4a894db1889d9dd01 [ 608.132043] env[62736]: INFO nova.scheduler.client.report [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Deleted allocations for instance 958e85c0-cb75-4dfd-92c4-b68957e87ec5 [ 608.138061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Expecting reply to msg 9e302daf1e34409d9a2bc404123fdff0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.148234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e302daf1e34409d9a2bc404123fdff0 [ 608.159800] env[62736]: DEBUG nova.network.neutron [-] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 608.160243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7012cf4f169440a0bfcbb33419e2f3a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.167883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7012cf4f169440a0bfcbb33419e2f3a9 [ 608.251546] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Releasing lock "refresh_cache-52f8e827-3faf-47c1-b793-ccacc5a0bc4c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 608.251813] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 608.251978] env[62736]: DEBUG nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 608.252161] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 608.272582] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 608.273217] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 2d4111f0f8124868952e8e09dec20476 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.281433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d4111f0f8124868952e8e09dec20476 [ 608.625419] env[62736]: DEBUG nova.scheduler.client.report [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 608.627847] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg a44f1b52b6ff4e1cbcade34e4cd8565f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.639106] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a44f1b52b6ff4e1cbcade34e4cd8565f [ 608.639662] env[62736]: DEBUG oslo_concurrency.lockutils [None req-662d052f-b537-404d-ba0d-b15485fa3d1f tempest-ImagesNegativeTestJSON-1008518018 tempest-ImagesNegativeTestJSON-1008518018-project-member] Lock "958e85c0-cb75-4dfd-92c4-b68957e87ec5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 74.190s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 608.640168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg e7cccce0581e46c6b094f6f61174354c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.649026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7cccce0581e46c6b094f6f61174354c [ 608.662450] env[62736]: INFO nova.compute.manager [-] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Took 1.21 seconds to deallocate network for instance. [ 608.664897] env[62736]: DEBUG nova.compute.claims [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 608.665069] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 608.781692] env[62736]: DEBUG nova.network.neutron [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 608.782222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 1837493f94c7492c9b5a75a52454fd01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 608.790787] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1837493f94c7492c9b5a75a52454fd01 [ 609.130451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.131s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 609.131581] env[62736]: ERROR nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Traceback (most recent call last): [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.driver.spawn(context, instance, image_meta, [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] vm_ref = self.build_virtual_machine(instance, [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] vif_infos = vmwarevif.get_vif_info(self._session, [ 609.131581] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] for vif in network_info: [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return self._sync_wrapper(fn, *args, **kwargs) [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.wait() [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self[:] = self._gt.wait() [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return self._exit_event.wait() [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] current.throw(*self._exc) [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 609.131927] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] result = function(*args, **kwargs) [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] return func(*args, **kwargs) [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise e [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] nwinfo = self.network_api.allocate_for_instance( [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] created_port_ids = self._update_ports_for_instance( [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] with excutils.save_and_reraise_exception(): [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] self.force_reraise() [ 609.132208] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise self.value [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] updated_port = self._update_port( [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] _ensure_no_port_binding_failure(port) [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] raise exception.PortBindingFailed(port_id=port['id']) [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] nova.exception.PortBindingFailed: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. [ 609.132460] env[62736]: ERROR nova.compute.manager [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] [ 609.132460] env[62736]: DEBUG nova.compute.utils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 609.133023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.584s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 609.134900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 7bcc3e23ad0246d1889fd011acd54154 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.136619] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Build of instance f537a3a9-b63a-4233-968a-942aca6ada0e was re-scheduled: Binding failed for port 3b1dae23-7093-47dd-9cbe-fa5054998b51, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 609.136990] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 609.137215] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquiring lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 609.137358] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Acquired lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 609.137514] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 609.137882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg fffbc46346f344519dc6351264f1b7fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.142960] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 609.144514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg be1d714a2c4a4342b55d18728b03e1b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.145503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fffbc46346f344519dc6351264f1b7fd [ 609.199858] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bcc3e23ad0246d1889fd011acd54154 [ 609.207063] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be1d714a2c4a4342b55d18728b03e1b3 [ 609.285015] env[62736]: INFO nova.compute.manager [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] [instance: 52f8e827-3faf-47c1-b793-ccacc5a0bc4c] Took 1.03 seconds to deallocate network for instance. [ 609.286982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 97afe4c530ce4f7d82469efe2a080f31 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.323043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97afe4c530ce4f7d82469efe2a080f31 [ 609.666813] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.686174] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 609.785939] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 609.785939] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg facb9007a8f04640a99e816068739c5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.790843] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg ef828173cc3e46cfa92246a207174b25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 609.798880] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg facb9007a8f04640a99e816068739c5c [ 609.832317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef828173cc3e46cfa92246a207174b25 [ 610.118221] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d3e8fd9-6c1b-44d5-aa38-3f49ffaa9e5c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.126156] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-169db836-614b-4c40-94e2-12824311fbb5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.162100] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75f380b0-6cb5-4478-925b-a40bec26c50e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.169278] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2df8ace-cc6a-4d01-8748-5af9a56c4c36 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.182117] env[62736]: DEBUG nova.compute.provider_tree [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 610.182628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 6429cfac4c704461a95febf24dd31f04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.189572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6429cfac4c704461a95febf24dd31f04 [ 610.289495] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Releasing lock "refresh_cache-f537a3a9-b63a-4233-968a-942aca6ada0e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 610.289809] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 610.290009] env[62736]: DEBUG nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 610.290195] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 610.312665] env[62736]: INFO nova.scheduler.client.report [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Deleted allocations for instance 52f8e827-3faf-47c1-b793-ccacc5a0bc4c [ 610.318627] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 610.319195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg ca8f811c64694e9b9ade791801400b1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.320923] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Expecting reply to msg 4f7e9b7275bc41309f1e587873e02ba5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.325682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca8f811c64694e9b9ade791801400b1d [ 610.353961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f7e9b7275bc41309f1e587873e02ba5 [ 610.688561] env[62736]: DEBUG nova.scheduler.client.report [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 610.688561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 4fe92220febd41079fe339fe564a72fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.706219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fe92220febd41079fe339fe564a72fa [ 610.824363] env[62736]: DEBUG nova.network.neutron [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 610.824363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 7e96319c250545bcb7d09e92a29b9dd3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.824760] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fcd30cc0-1946-4ab4-a017-ab01550a2c99 tempest-ServerExternalEventsTest-1600310199 tempest-ServerExternalEventsTest-1600310199-project-member] Lock "52f8e827-3faf-47c1-b793-ccacc5a0bc4c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 73.902s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 610.825240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg adfb33d526a842f2a97d6fabb1fca1ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 610.834073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e96319c250545bcb7d09e92a29b9dd3 [ 610.835497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adfb33d526a842f2a97d6fabb1fca1ea [ 611.190194] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.057s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 611.190895] env[62736]: ERROR nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Traceback (most recent call last): [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.driver.spawn(context, instance, image_meta, [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] vm_ref = self.build_virtual_machine(instance, [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] vif_infos = vmwarevif.get_vif_info(self._session, [ 611.190895] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] for vif in network_info: [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self._sync_wrapper(fn, *args, **kwargs) [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.wait() [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self[:] = self._gt.wait() [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self._exit_event.wait() [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] result = hub.switch() [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 611.191172] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return self.greenlet.switch() [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] result = function(*args, **kwargs) [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] return func(*args, **kwargs) [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise e [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] nwinfo = self.network_api.allocate_for_instance( [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] created_port_ids = self._update_ports_for_instance( [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] with excutils.save_and_reraise_exception(): [ 611.191478] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] self.force_reraise() [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise self.value [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] updated_port = self._update_port( [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] _ensure_no_port_binding_failure(port) [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] raise exception.PortBindingFailed(port_id=port['id']) [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] nova.exception.PortBindingFailed: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. [ 611.191792] env[62736]: ERROR nova.compute.manager [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] [ 611.192071] env[62736]: DEBUG nova.compute.utils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 611.192814] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.963s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.194546] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 8fb23b306fed4a6fbc777f67680f2277 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.200035] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Build of instance bd41759b-5c32-42e6-b625-4ea48f98045e was re-scheduled: Binding failed for port 74c43113-48d4-426b-9dd2-405b9e6ab39f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 611.200035] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 611.200035] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquiring lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.200035] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Acquired lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.200347] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 611.200347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 66c35a1e19ad48fb9ccb86cfc857ca8c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.203571] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66c35a1e19ad48fb9ccb86cfc857ca8c [ 611.228400] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fb23b306fed4a6fbc777f67680f2277 [ 611.329621] env[62736]: INFO nova.compute.manager [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] [instance: f537a3a9-b63a-4233-968a-942aca6ada0e] Took 1.04 seconds to deallocate network for instance. [ 611.331335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg ff491f443d5d4579ab5d0e36cfb9342a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.332361] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 611.333817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 2b4d6d570cd74d709bbe698d39950932 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.371781] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b4d6d570cd74d709bbe698d39950932 [ 611.372582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff491f443d5d4579ab5d0e36cfb9342a [ 611.718482] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 611.815996] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.816554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg e1a838e086fe4a5db7e8646d3106ed46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.826118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1a838e086fe4a5db7e8646d3106ed46 [ 611.840407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg b24f7defaf8b46f89b183b58f38bdb4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.858658] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.876858] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b24f7defaf8b46f89b183b58f38bdb4d [ 611.980670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d0f77d4a25374591bc236473d772fa1a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 611.992034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0f77d4a25374591bc236473d772fa1a [ 612.121306] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-700ed72d-86ea-4e52-97dd-d00998eb2743 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.129964] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62f14722-2f3e-466f-b6bc-40f20c797d2b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.167093] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8f419ac-31b6-4196-91e9-9fffe22cdd3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.174306] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ef731d9-089c-40bd-a17f-4747e751f354 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.188496] env[62736]: DEBUG nova.compute.provider_tree [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 612.188998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 81d6c21e70c64651be5e97ae16dc9a3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.196516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81d6c21e70c64651be5e97ae16dc9a3a [ 612.324591] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Releasing lock "refresh_cache-bd41759b-5c32-42e6-b625-4ea48f98045e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 612.324591] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 612.324591] env[62736]: DEBUG nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 612.324591] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 612.335766] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 612.336426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 5dab1f245627439ea14019582427c909 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.344023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5dab1f245627439ea14019582427c909 [ 612.372126] env[62736]: INFO nova.scheduler.client.report [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Deleted allocations for instance f537a3a9-b63a-4233-968a-942aca6ada0e [ 612.377432] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Expecting reply to msg 0f91ccd6369b4cacba4800d6b85b3117 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.394628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f91ccd6369b4cacba4800d6b85b3117 [ 612.691482] env[62736]: DEBUG nova.scheduler.client.report [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 612.693951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 904ce8cff22441ffade1aab14de976a5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.707334] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 904ce8cff22441ffade1aab14de976a5 [ 612.838634] env[62736]: DEBUG nova.network.neutron [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 612.838717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 10a81f0de20b42b9b3096146e18a06c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.847579] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10a81f0de20b42b9b3096146e18a06c8 [ 612.879006] env[62736]: DEBUG oslo_concurrency.lockutils [None req-098d58f7-87c6-4589-b09b-42ae718b38e8 tempest-TenantUsagesTestJSON-2051145345 tempest-TenantUsagesTestJSON-2051145345-project-member] Lock "f537a3a9-b63a-4233-968a-942aca6ada0e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 74.199s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 612.879663] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 351cb056a4754d0290644013899429ad in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 612.896927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 351cb056a4754d0290644013899429ad [ 613.197452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.004s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.198083] env[62736]: ERROR nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Traceback (most recent call last): [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.driver.spawn(context, instance, image_meta, [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] vm_ref = self.build_virtual_machine(instance, [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] vif_infos = vmwarevif.get_vif_info(self._session, [ 613.198083] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] for vif in network_info: [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return self._sync_wrapper(fn, *args, **kwargs) [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.wait() [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self[:] = self._gt.wait() [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return self._exit_event.wait() [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] current.throw(*self._exc) [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 613.198356] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] result = function(*args, **kwargs) [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] return func(*args, **kwargs) [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise e [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] nwinfo = self.network_api.allocate_for_instance( [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] created_port_ids = self._update_ports_for_instance( [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] with excutils.save_and_reraise_exception(): [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] self.force_reraise() [ 613.198643] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise self.value [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] updated_port = self._update_port( [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] _ensure_no_port_binding_failure(port) [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] raise exception.PortBindingFailed(port_id=port['id']) [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] nova.exception.PortBindingFailed: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. [ 613.198940] env[62736]: ERROR nova.compute.manager [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] [ 613.198940] env[62736]: DEBUG nova.compute.utils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 613.200028] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.478s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.202419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg fae5914999e2414cba8926242c142d11 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.203612] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Build of instance 25508ff0-bc62-47b3-ae9e-7d9d8c735edc was re-scheduled: Binding failed for port db4fac68-33e9-43e4-a979-d71a08eb0e06, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 613.204056] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 613.204290] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquiring lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.204429] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Acquired lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.204575] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 613.204920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg e647fd31c80e415bbce9d684434e47a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.213471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e647fd31c80e415bbce9d684434e47a0 [ 613.249607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fae5914999e2414cba8926242c142d11 [ 613.341218] env[62736]: INFO nova.compute.manager [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] [instance: bd41759b-5c32-42e6-b625-4ea48f98045e] Took 1.02 seconds to deallocate network for instance. [ 613.343037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 524e9ae0586b437da84ff7ea598e7807 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.382865] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 613.385122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8b446de7c5e340ff9fb96102cc2d8e97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.390795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 524e9ae0586b437da84ff7ea598e7807 [ 613.417573] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b446de7c5e340ff9fb96102cc2d8e97 [ 613.745000] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.848395] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 0e58e52ea2aa4e0f913a243be7dc902f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.880386] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.880863] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 2018c3f5f1bc4112af1373cb424708eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 613.892899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2018c3f5f1bc4112af1373cb424708eb [ 613.899876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e58e52ea2aa4e0f913a243be7dc902f [ 613.901678] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 614.204734] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80dbb3bb-43fd-4215-bf92-88cdf5183382 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.220320] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e12603ff-ee7f-476d-9cd7-db92894076f0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.254898] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca5c602f-4a42-4c89-b64f-da5d68291873 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.262812] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cd7c7b-1301-4e80-97d5-b26b01e17c6e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 614.276387] env[62736]: DEBUG nova.compute.provider_tree [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 614.277007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg d9c96f226e7747949d0db8418a8323d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.285926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9c96f226e7747949d0db8418a8323d6 [ 614.376368] env[62736]: INFO nova.scheduler.client.report [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Deleted allocations for instance bd41759b-5c32-42e6-b625-4ea48f98045e [ 614.390904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Expecting reply to msg 1bb1ca2d468d47f082a3901092d492c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.390904] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Releasing lock "refresh_cache-25508ff0-bc62-47b3-ae9e-7d9d8c735edc" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 614.390904] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 614.390904] env[62736]: DEBUG nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 614.390904] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 614.395858] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bb1ca2d468d47f082a3901092d492c7 [ 614.415673] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 614.416421] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 00d9db1b1ba049b7bee5a10c2722b3ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.427766] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00d9db1b1ba049b7bee5a10c2722b3ae [ 614.795613] env[62736]: DEBUG nova.scheduler.client.report [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 614.795613] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg e769e089a7af46188e8f1034f59ecd7e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.807485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e769e089a7af46188e8f1034f59ecd7e [ 614.896952] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2cbeab54-ce31-4fd2-a93e-cef8b9a60b8c tempest-VolumesAssistedSnapshotsTest-161195681 tempest-VolumesAssistedSnapshotsTest-161195681-project-member] Lock "bd41759b-5c32-42e6-b625-4ea48f98045e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 72.673s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 614.897569] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg d37e0375c660426b9adbd0c9580f395d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.912519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d37e0375c660426b9adbd0c9580f395d [ 614.919343] env[62736]: DEBUG nova.network.neutron [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.919343] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg d3d7a46654ed4b28838360b123103006 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 614.926707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3d7a46654ed4b28838360b123103006 [ 615.292681] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.092s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 615.293288] env[62736]: ERROR nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Traceback (most recent call last): [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.driver.spawn(context, instance, image_meta, [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] vm_ref = self.build_virtual_machine(instance, [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] vif_infos = vmwarevif.get_vif_info(self._session, [ 615.293288] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] for vif in network_info: [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self._sync_wrapper(fn, *args, **kwargs) [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.wait() [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self[:] = self._gt.wait() [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self._exit_event.wait() [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] result = hub.switch() [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 615.293566] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return self.greenlet.switch() [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] result = function(*args, **kwargs) [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] return func(*args, **kwargs) [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise e [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] nwinfo = self.network_api.allocate_for_instance( [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] created_port_ids = self._update_ports_for_instance( [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] with excutils.save_and_reraise_exception(): [ 615.293946] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] self.force_reraise() [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise self.value [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] updated_port = self._update_port( [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] _ensure_no_port_binding_failure(port) [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] raise exception.PortBindingFailed(port_id=port['id']) [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] nova.exception.PortBindingFailed: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. [ 615.294237] env[62736]: ERROR nova.compute.manager [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] [ 615.294500] env[62736]: DEBUG nova.compute.utils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 615.295998] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.392s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 615.298206] env[62736]: INFO nova.compute.claims [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 615.306043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg bb141d6476104d9d9f63e322bde353a4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.307417] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Build of instance 918aa256-a10d-46d1-b9a4-ecd70cf3566f was re-scheduled: Binding failed for port d1d9c4d7-2cd5-43d7-8471-60c524b7cfda, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 615.307890] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 615.308139] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquiring lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 615.308282] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Acquired lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 615.308968] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 615.308968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg ddece5841709453097eb14b382a3b8d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.323576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddece5841709453097eb14b382a3b8d6 [ 615.351465] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb141d6476104d9d9f63e322bde353a4 [ 615.399672] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 615.401591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg c2b7e2d9041941478f854a487ec1b8c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.421222] env[62736]: INFO nova.compute.manager [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] [instance: 25508ff0-bc62-47b3-ae9e-7d9d8c735edc] Took 1.03 seconds to deallocate network for instance. [ 615.422877] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 18555c8943c447558836b3fc979278c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.447332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2b7e2d9041941478f854a487ec1b8c4 [ 615.472051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18555c8943c447558836b3fc979278c7 [ 615.812554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 74a0f42b6d704bfca84bd26f7b53ea77 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.822994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74a0f42b6d704bfca84bd26f7b53ea77 [ 615.846677] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 615.927258] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 0de1128a534d4ab89d89a74274ce5d81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 615.933635] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 615.965761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0de1128a534d4ab89d89a74274ce5d81 [ 616.120913] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 616.121448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg f543a736783247fca1702a9ec9a0dbcb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 616.131135] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f543a736783247fca1702a9ec9a0dbcb [ 616.453960] env[62736]: INFO nova.scheduler.client.report [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Deleted allocations for instance 25508ff0-bc62-47b3-ae9e-7d9d8c735edc [ 616.460147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Expecting reply to msg 51fad7b60bc44f39aa1bc98d09ccb88d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 616.478018] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51fad7b60bc44f39aa1bc98d09ccb88d [ 616.623254] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Releasing lock "refresh_cache-918aa256-a10d-46d1-b9a4-ecd70cf3566f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 616.623513] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 616.623698] env[62736]: DEBUG nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 616.623860] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 616.647268] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 616.648209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 8469161a852b4c51ab631096954de65b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 616.659738] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8469161a852b4c51ab631096954de65b [ 616.768691] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-165f382e-87f1-4a8f-b6ba-6ece429cb19d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.776717] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16163dde-02f2-4edd-869a-2d5eb5cff47f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.821115] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12c1afdd-8a50-49de-9b28-118db4dd8dfe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.830489] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d5194c9-e2b2-44f2-a9fa-576e1ef7befd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 616.846230] env[62736]: DEBUG nova.compute.provider_tree [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 616.846775] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d2c7a8e25b71498a8291770bc50bc397 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 616.855958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2c7a8e25b71498a8291770bc50bc397 [ 616.961704] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b05dba1a-f8ea-4623-ac42-9defb01479a1 tempest-InstanceActionsTestJSON-1210756679 tempest-InstanceActionsTestJSON-1210756679-project-member] Lock "25508ff0-bc62-47b3-ae9e-7d9d8c735edc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 73.977s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.962830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 565c8fad646c4308936905c820dde157 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 616.974775] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 565c8fad646c4308936905c820dde157 [ 617.151021] env[62736]: DEBUG nova.network.neutron [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.151808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 2187838b3a444ba49f3f6a5522baced9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.159969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2187838b3a444ba49f3f6a5522baced9 [ 617.353729] env[62736]: DEBUG nova.scheduler.client.report [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 617.353729] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 3d2c38e9546b4645af5623a8f8a7ad46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.367230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d2c38e9546b4645af5623a8f8a7ad46 [ 617.467159] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 617.467159] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 1247fcf7ea2e4698b1faf504fc882219 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.504489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1247fcf7ea2e4698b1faf504fc882219 [ 617.655502] env[62736]: INFO nova.compute.manager [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] [instance: 918aa256-a10d-46d1-b9a4-ecd70cf3566f] Took 1.03 seconds to deallocate network for instance. [ 617.657542] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 5108de9c97824aadad38f25c043a35e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.710823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5108de9c97824aadad38f25c043a35e1 [ 617.860203] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.560s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.860203] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 617.860203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 97ec8556fecc4e77af539e716808f0de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.860203] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.211s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.864186] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 59f0eaaba64144bfbfb754135bf14959 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 617.896836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59f0eaaba64144bfbfb754135bf14959 [ 617.898229] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97ec8556fecc4e77af539e716808f0de [ 617.993627] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.162354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 29c1067e3ddf45088141ab09f30c43b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 618.206695] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29c1067e3ddf45088141ab09f30c43b4 [ 618.365912] env[62736]: DEBUG nova.compute.utils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 618.366588] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 8a25d0eff50444ba9b8dfdab73239a7d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 618.371575] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 618.371575] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 618.377878] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a25d0eff50444ba9b8dfdab73239a7d [ 618.529599] env[62736]: DEBUG nova.policy [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c43c33165931465494c0a7da171126e4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de11ac365f704c2986c782a5918f802f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 618.690481] env[62736]: INFO nova.scheduler.client.report [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Deleted allocations for instance 918aa256-a10d-46d1-b9a4-ecd70cf3566f [ 618.701292] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Expecting reply to msg 8ee9a3bbc973491fbcef2657d6184c55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 618.723295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ee9a3bbc973491fbcef2657d6184c55 [ 618.811562] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe7b1738-0413-4603-b42b-b6791645743d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.819438] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a22daa52-2148-4f59-ac1c-f8443406ac47 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.853814] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a60fca13-468a-4d4d-9bde-6793ade2b31d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.861352] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b56525f-06e6-410f-a928-3f8e78aeadba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 618.875043] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 618.878080] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 1431a440c345489f8dcfe2925a216827 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 618.878080] env[62736]: DEBUG nova.compute.provider_tree [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 618.879161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 17ebfeb801b5469eb886c08a924f670f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 618.886836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17ebfeb801b5469eb886c08a924f670f [ 618.921844] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1431a440c345489f8dcfe2925a216827 [ 619.077056] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Successfully created port: fa29c075-7ce5-482f-8388-05891144179b {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 619.203397] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3e62278-36b5-4c99-9cae-928a9ac51e8f tempest-ServerActionsTestOtherB-819696560 tempest-ServerActionsTestOtherB-819696560-project-member] Lock "918aa256-a10d-46d1-b9a4-ecd70cf3566f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 70.917s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.204044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 5516a3ba62e14ebaa36a976be99d87f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.218026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5516a3ba62e14ebaa36a976be99d87f3 [ 619.391940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 9e743cd7bc4942f0833ee95890b697e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.391940] env[62736]: DEBUG nova.scheduler.client.report [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 619.391940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 759aecdd4040404ab7234c0eba2b737f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.410224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 759aecdd4040404ab7234c0eba2b737f [ 619.421511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e743cd7bc4942f0833ee95890b697e3 [ 619.706659] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 619.708554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 7ebf53e2c44a486d983f7828eab17e19 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.772763] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ebf53e2c44a486d983f7828eab17e19 [ 619.891996] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 619.892545] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.032s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.893256] env[62736]: ERROR nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Traceback (most recent call last): [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.driver.spawn(context, instance, image_meta, [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self._vmops.spawn(context, instance, image_meta, injected_files, [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] vm_ref = self.build_virtual_machine(instance, [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] vif_infos = vmwarevif.get_vif_info(self._session, [ 619.893256] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] for vif in network_info: [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self._sync_wrapper(fn, *args, **kwargs) [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.wait() [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self[:] = self._gt.wait() [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self._exit_event.wait() [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] result = hub.switch() [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 619.893555] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return self.greenlet.switch() [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] result = function(*args, **kwargs) [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] return func(*args, **kwargs) [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise e [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] nwinfo = self.network_api.allocate_for_instance( [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] created_port_ids = self._update_ports_for_instance( [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] with excutils.save_and_reraise_exception(): [ 619.893849] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] self.force_reraise() [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise self.value [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] updated_port = self._update_port( [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] _ensure_no_port_binding_failure(port) [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] raise exception.PortBindingFailed(port_id=port['id']) [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] nova.exception.PortBindingFailed: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. [ 619.894129] env[62736]: ERROR nova.compute.manager [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] [ 619.894416] env[62736]: DEBUG nova.compute.utils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 619.895162] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.195s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.896992] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 309611f2d1ec4be2ba02fab3b7111e0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.900058] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Build of instance 64ee370c-6d43-4e2b-a27a-3aa716fdc322 was re-scheduled: Binding failed for port 2fc1b9dc-5720-4a4e-8ecd-4e25c684be47, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 619.900058] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 619.900058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquiring lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 619.900058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Acquired lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 619.900392] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 619.900392] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 1b51ccdfedbd4a53b6927bfa9b72c0b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 619.909857] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b51ccdfedbd4a53b6927bfa9b72c0b2 [ 619.920686] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 619.920946] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 619.921100] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 619.921275] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 619.921726] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 619.921726] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 619.921832] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 619.921892] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 619.922049] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 619.922208] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 619.922376] env[62736]: DEBUG nova.virt.hardware [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 619.923522] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b422b0a3-d544-42cd-a05b-dccf2e3c6caf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.934662] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6568033-0d62-4b8d-8e37-01cb4482a5b5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 619.946583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 309611f2d1ec4be2ba02fab3b7111e0e [ 620.242106] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 620.437966] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 620.788655] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 620.789170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 8ff4398f3f754293b64d71420766f3fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 620.802512] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ff4398f3f754293b64d71420766f3fe [ 620.858163] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be54dffe-0602-4db5-be51-21b776a917c9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.866729] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-298ef849-03a0-4ea9-af39-a79ea1517257 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.914848] env[62736]: ERROR nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 620.914848] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 620.914848] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 620.914848] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 620.914848] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 620.914848] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 620.914848] env[62736]: ERROR nova.compute.manager raise self.value [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 620.914848] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 620.914848] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 620.914848] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 620.915256] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 620.915256] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 620.915256] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 620.915256] env[62736]: ERROR nova.compute.manager [ 620.915256] env[62736]: Traceback (most recent call last): [ 620.915256] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 620.915256] env[62736]: listener.cb(fileno) [ 620.915256] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 620.915256] env[62736]: result = function(*args, **kwargs) [ 620.915256] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 620.915256] env[62736]: return func(*args, **kwargs) [ 620.915256] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 620.915256] env[62736]: raise e [ 620.915256] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 620.915256] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 620.915256] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 620.915256] env[62736]: created_port_ids = self._update_ports_for_instance( [ 620.915256] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 620.915256] env[62736]: with excutils.save_and_reraise_exception(): [ 620.915256] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 620.915256] env[62736]: self.force_reraise() [ 620.915256] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 620.915256] env[62736]: raise self.value [ 620.915256] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 620.915256] env[62736]: updated_port = self._update_port( [ 620.915256] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 620.915256] env[62736]: _ensure_no_port_binding_failure(port) [ 620.915256] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 620.915256] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 620.915961] env[62736]: nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 620.915961] env[62736]: Removing descriptor: 16 [ 620.917006] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2b910e8-8d7c-451d-ab49-a4c55fbeb6de {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.920780] env[62736]: DEBUG nova.compute.manager [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Received event network-changed-fa29c075-7ce5-482f-8388-05891144179b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 620.920954] env[62736]: DEBUG nova.compute.manager [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Refreshing instance network info cache due to event network-changed-fa29c075-7ce5-482f-8388-05891144179b. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 620.921160] env[62736]: DEBUG oslo_concurrency.lockutils [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] Acquiring lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 620.921294] env[62736]: DEBUG oslo_concurrency.lockutils [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] Acquired lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 620.921454] env[62736]: DEBUG nova.network.neutron [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Refreshing network info cache for port fa29c075-7ce5-482f-8388-05891144179b {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 620.921883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] Expecting reply to msg fe973be7b0734093b74a4741ef2e43bc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 620.923382] env[62736]: ERROR nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] Traceback (most recent call last): [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] yield resources [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.driver.spawn(context, instance, image_meta, [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self._vmops.spawn(context, instance, image_meta, injected_files, [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] vm_ref = self.build_virtual_machine(instance, [ 620.923382] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] vif_infos = vmwarevif.get_vif_info(self._session, [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] for vif in network_info: [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self._sync_wrapper(fn, *args, **kwargs) [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.wait() [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self[:] = self._gt.wait() [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self._exit_event.wait() [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 620.923679] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] result = hub.switch() [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self.greenlet.switch() [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] result = function(*args, **kwargs) [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return func(*args, **kwargs) [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise e [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] nwinfo = self.network_api.allocate_for_instance( [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] created_port_ids = self._update_ports_for_instance( [ 620.924031] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] with excutils.save_and_reraise_exception(): [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.force_reraise() [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise self.value [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] updated_port = self._update_port( [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] _ensure_no_port_binding_failure(port) [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise exception.PortBindingFailed(port_id=port['id']) [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 620.924323] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] [ 620.924611] env[62736]: INFO nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Terminating instance [ 620.930099] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 620.935107] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe973be7b0734093b74a4741ef2e43bc [ 620.936940] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67d090de-a22d-4c86-af24-22d9f3f684f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 620.952530] env[62736]: DEBUG nova.compute.provider_tree [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 620.953346] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 8634d996d02d449aa0433b8b2a08fd9d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 620.964433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8634d996d02d449aa0433b8b2a08fd9d [ 621.293569] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Releasing lock "refresh_cache-64ee370c-6d43-4e2b-a27a-3aa716fdc322" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 621.293823] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 621.294006] env[62736]: DEBUG nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 621.294162] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 621.320529] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 621.321122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 3770a43138f9469fb17f1a97a98a1352 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.327764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3770a43138f9469fb17f1a97a98a1352 [ 621.455819] env[62736]: DEBUG nova.scheduler.client.report [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 621.458359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 2d356604d1d94fb48ea7ed93dd960737 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.469566] env[62736]: DEBUG nova.network.neutron [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 621.474315] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d356604d1d94fb48ea7ed93dd960737 [ 621.654962] env[62736]: DEBUG nova.network.neutron [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 621.655512] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] Expecting reply to msg 9ab85064930b4eb39b4dc88ce2ba3f0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.664732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ab85064930b4eb39b4dc88ce2ba3f0e [ 621.824193] env[62736]: DEBUG nova.network.neutron [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 621.824746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 381388eab6cf418c8edadffb240442d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.836797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 381388eab6cf418c8edadffb240442d1 [ 621.962034] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.067s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 621.962699] env[62736]: ERROR nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] Traceback (most recent call last): [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.driver.spawn(context, instance, image_meta, [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] vm_ref = self.build_virtual_machine(instance, [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] vif_infos = vmwarevif.get_vif_info(self._session, [ 621.962699] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] for vif in network_info: [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self._sync_wrapper(fn, *args, **kwargs) [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.wait() [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self[:] = self._gt.wait() [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self._exit_event.wait() [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] result = hub.switch() [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 621.963019] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return self.greenlet.switch() [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] result = function(*args, **kwargs) [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] return func(*args, **kwargs) [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise e [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] nwinfo = self.network_api.allocate_for_instance( [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] created_port_ids = self._update_ports_for_instance( [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] with excutils.save_and_reraise_exception(): [ 621.963483] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] self.force_reraise() [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise self.value [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] updated_port = self._update_port( [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] _ensure_no_port_binding_failure(port) [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] raise exception.PortBindingFailed(port_id=port['id']) [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] nova.exception.PortBindingFailed: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. [ 621.963860] env[62736]: ERROR nova.compute.manager [instance: 3a848010-382b-4753-8420-37222963e7e2] [ 621.964146] env[62736]: DEBUG nova.compute.utils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 621.964693] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.300s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 621.966648] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 08aa39d68b9d4695a23481f78d2c117a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.979202] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Build of instance 3a848010-382b-4753-8420-37222963e7e2 was re-scheduled: Binding failed for port ad867a88-c3d1-405e-be2f-5e3328b9c9b9, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 621.979202] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 621.979202] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquiring lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 621.979202] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Acquired lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 621.979469] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 621.979469] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 191c65a5d7cb4e7ca114be470202cec6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 621.985045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 191c65a5d7cb4e7ca114be470202cec6 [ 622.009962] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08aa39d68b9d4695a23481f78d2c117a [ 622.158422] env[62736]: DEBUG oslo_concurrency.lockutils [req-2530af7e-e74d-4e95-8e45-6fdfdf5bc65a req-457f1bb8-39d1-4583-b3a4-f40f7a137227 service nova] Releasing lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 622.158422] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 622.158978] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 622.160476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 29c69bd7e9824a6a992570a65739a2b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 622.168240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29c69bd7e9824a6a992570a65739a2b4 [ 622.329526] env[62736]: INFO nova.compute.manager [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] [instance: 64ee370c-6d43-4e2b-a27a-3aa716fdc322] Took 1.03 seconds to deallocate network for instance. [ 622.329526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 87f2c67e83504dc99aa6cc8e021e8cdc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 622.371003] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87f2c67e83504dc99aa6cc8e021e8cdc [ 622.504122] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 622.666920] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 622.667807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 8ac72884a5f04442b5f59b17c89024ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 622.680411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ac72884a5f04442b5f59b17c89024ca [ 622.709129] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 622.834896] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 399f7510032749e19f5ed3e4259aa8bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 622.888068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 399f7510032749e19f5ed3e4259aa8bf [ 622.941590] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba9187ed-d1e4-4d74-954a-80e5c2bc0307 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.949321] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ffb1b7d-25af-4d6c-b681-c113cc1123b9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.982164] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4cc1e82-eeb1-4a76-bf71-145643f5e353 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 622.990820] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-829150f6-b3c9-470f-a69b-6327b59de8af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.012786] env[62736]: DEBUG nova.compute.provider_tree [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 623.013683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 614fc9c884da4243ab402fd77a32e3c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.020985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 614fc9c884da4243ab402fd77a32e3c1 [ 623.063146] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 623.063146] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 40b18c33cee24512b210ab5250c837b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.073074] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40b18c33cee24512b210ab5250c837b7 [ 623.174275] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Releasing lock "refresh_cache-3a848010-382b-4753-8420-37222963e7e2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 623.174275] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 623.174275] env[62736]: DEBUG nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 623.174275] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 623.209727] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 623.210335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg b99c0b538d2b417aa6bb506a5d9e1376 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.220675] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b99c0b538d2b417aa6bb506a5d9e1376 [ 623.370855] env[62736]: INFO nova.scheduler.client.report [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Deleted allocations for instance 64ee370c-6d43-4e2b-a27a-3aa716fdc322 [ 623.381547] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Expecting reply to msg 24c1b2ffedbf4c2abf0bf89406bf3688 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.397998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24c1b2ffedbf4c2abf0bf89406bf3688 [ 623.516611] env[62736]: DEBUG nova.scheduler.client.report [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 623.519050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 7560af2654b04fed98b086a6f29e30ba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.526478] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquiring lock "427400bc-f265-40cb-8c1c-ccab7bc94b1c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 623.526478] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "427400bc-f265-40cb-8c1c-ccab7bc94b1c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 623.534376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7560af2654b04fed98b086a6f29e30ba [ 623.573433] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 623.573873] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 623.574060] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 623.574344] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-84898e06-4b96-48c9-bcd7-a12be857d720 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.582802] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8b199db-77a5-4ba2-a131-884aba6f21d3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 623.603847] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 558058e0-9808-415a-8866-75d1c8598d43 could not be found. [ 623.604071] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 623.604251] env[62736]: INFO nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Took 0.03 seconds to destroy the instance on the hypervisor. [ 623.604488] env[62736]: DEBUG oslo.service.loopingcall [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 623.604699] env[62736]: DEBUG nova.compute.manager [-] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 623.604790] env[62736]: DEBUG nova.network.neutron [-] [instance: 558058e0-9808-415a-8866-75d1c8598d43] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 623.712223] env[62736]: DEBUG nova.network.neutron [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 623.712987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg d92caa4458cf430caa68f4e62759c833 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.720726] env[62736]: DEBUG nova.network.neutron [-] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 623.720726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2ce39db48d004eab90c7f738fab6a206 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.721524] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d92caa4458cf430caa68f4e62759c833 [ 623.726828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ce39db48d004eab90c7f738fab6a206 [ 623.752146] env[62736]: DEBUG nova.compute.manager [req-12381b09-8fb8-4ba3-8bcb-dfb6ae60b71a req-c2847185-fccb-4129-8590-eeeb3d955782 service nova] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Received event network-vif-deleted-fa29c075-7ce5-482f-8388-05891144179b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 623.883147] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d8ab2ec-6616-469b-bdfc-cefe695f9c27 tempest-ServersWithSpecificFlavorTestJSON-1147380317 tempest-ServersWithSpecificFlavorTestJSON-1147380317-project-member] Lock "64ee370c-6d43-4e2b-a27a-3aa716fdc322" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 74.442s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 623.883772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg fdbb022230ed449787063341ade8a86b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 623.899886] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdbb022230ed449787063341ade8a86b [ 624.021753] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.057s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 624.022411] env[62736]: ERROR nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Traceback (most recent call last): [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.driver.spawn(context, instance, image_meta, [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] vm_ref = self.build_virtual_machine(instance, [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] vif_infos = vmwarevif.get_vif_info(self._session, [ 624.022411] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] for vif in network_info: [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self._sync_wrapper(fn, *args, **kwargs) [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.wait() [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self[:] = self._gt.wait() [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self._exit_event.wait() [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] result = hub.switch() [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 624.022690] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return self.greenlet.switch() [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] result = function(*args, **kwargs) [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] return func(*args, **kwargs) [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise e [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] nwinfo = self.network_api.allocate_for_instance( [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] created_port_ids = self._update_ports_for_instance( [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] with excutils.save_and_reraise_exception(): [ 624.023016] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] self.force_reraise() [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise self.value [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] updated_port = self._update_port( [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] _ensure_no_port_binding_failure(port) [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] raise exception.PortBindingFailed(port_id=port['id']) [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] nova.exception.PortBindingFailed: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. [ 624.023325] env[62736]: ERROR nova.compute.manager [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] [ 624.023707] env[62736]: DEBUG nova.compute.utils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 624.024543] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.365s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 624.026132] env[62736]: INFO nova.compute.claims [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 624.027740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg ba3d22e05a3a4e9e8c101495a200e041 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.028921] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Build of instance c7670280-b5a7-4de2-8cf9-6814a81eb1a6 was re-scheduled: Binding failed for port 4a3a996d-9a97-4d00-a63e-71097cd065b2, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 624.029397] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 624.029629] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquiring lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 624.029772] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Acquired lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 624.029925] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 624.030309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 824333cbbdf34facb3693294b1d26a07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.037286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 824333cbbdf34facb3693294b1d26a07 [ 624.071138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba3d22e05a3a4e9e8c101495a200e041 [ 624.217792] env[62736]: INFO nova.compute.manager [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] [instance: 3a848010-382b-4753-8420-37222963e7e2] Took 1.04 seconds to deallocate network for instance. [ 624.217792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg c99139a95bc44534aefa5c04690c6ffd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.222350] env[62736]: DEBUG nova.network.neutron [-] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.222350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b8f8e77f43e64df89ec70580a9f13e46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.231161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8f8e77f43e64df89ec70580a9f13e46 [ 624.259748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c99139a95bc44534aefa5c04690c6ffd [ 624.385755] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 624.387844] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 88cb0755754741278afb2950947ebc1b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.425485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88cb0755754741278afb2950947ebc1b [ 624.535457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg d8083da6092445e88974702bd4f1d821 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.543083] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8083da6092445e88974702bd4f1d821 [ 624.600948] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 624.722467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg 2509f10b5cb644f79e1626fb0cd1a7c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.725379] env[62736]: INFO nova.compute.manager [-] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Took 1.12 seconds to deallocate network for instance. [ 624.729057] env[62736]: DEBUG nova.compute.claims [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 624.729314] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 624.762586] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2509f10b5cb644f79e1626fb0cd1a7c7 [ 624.853144] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 624.853144] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg c291706c831c4797aae1bdd0746028b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 624.861733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c291706c831c4797aae1bdd0746028b7 [ 624.909988] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.256841] env[62736]: INFO nova.scheduler.client.report [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Deleted allocations for instance 3a848010-382b-4753-8420-37222963e7e2 [ 625.264902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Expecting reply to msg ec260e07a8fb4b0dbe5bf0d76011cc43 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 625.287922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec260e07a8fb4b0dbe5bf0d76011cc43 [ 625.362337] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Releasing lock "refresh_cache-c7670280-b5a7-4de2-8cf9-6814a81eb1a6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 625.362614] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 625.362803] env[62736]: DEBUG nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 625.362963] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 625.414351] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 625.415031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg ed594e0637dc449cbe3ff5c9ea115fe2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 625.423439] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed594e0637dc449cbe3ff5c9ea115fe2 [ 625.492438] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c3dbe67-92c0-4d10-b79a-04b114f82817 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.496635] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7539a618-3514-477b-bb02-d04f9cc04bac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.536916] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea6e8163-37fd-4401-af38-77e9dc672e07 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.553080] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-506a6d02-682a-48fe-9f5c-dbe82cbb20ed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 625.573673] env[62736]: DEBUG nova.compute.provider_tree [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 625.574178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 962e1d7554ab496fa811ccf2f0917009 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 625.585968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 962e1d7554ab496fa811ccf2f0917009 [ 625.769775] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c171efca-4e75-4a68-8b69-3d25d17b5305 tempest-FloatingIPsAssociationTestJSON-1881166845 tempest-FloatingIPsAssociationTestJSON-1881166845-project-member] Lock "3a848010-382b-4753-8420-37222963e7e2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 70.388s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.770378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 1f0a611573c54175917febf54e29b9ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 625.786251] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f0a611573c54175917febf54e29b9ff [ 625.919833] env[62736]: DEBUG nova.network.neutron [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.920314] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg b63292df33754db4bab60fff434fd3a2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 625.928651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b63292df33754db4bab60fff434fd3a2 [ 626.077608] env[62736]: DEBUG nova.scheduler.client.report [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 626.080083] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 0d1a1506054040d7ae8729692a97e455 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.096364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d1a1506054040d7ae8729692a97e455 [ 626.273183] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 626.275575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 469bfacc06f646149d82994db8122b9b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.309106] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 469bfacc06f646149d82994db8122b9b [ 626.425462] env[62736]: INFO nova.compute.manager [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] [instance: c7670280-b5a7-4de2-8cf9-6814a81eb1a6] Took 1.06 seconds to deallocate network for instance. [ 626.425462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 603c3de557e64836a6e7b2734576c16d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.471680] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 603c3de557e64836a6e7b2734576c16d [ 626.588038] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.558s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.588038] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 626.588038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 2e1412cca3e949b99a79e70b8ac20468 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.588038] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.727s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.588038] env[62736]: INFO nova.compute.claims [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 626.588801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 0f193bd168cd4dc4b9590fb6e516e79b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.629029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e1412cca3e949b99a79e70b8ac20468 [ 626.632062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f193bd168cd4dc4b9590fb6e516e79b [ 626.804267] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.930210] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg a437cc76606a404d84d8d02018c3d7c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 626.979666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a437cc76606a404d84d8d02018c3d7c3 [ 627.092247] env[62736]: DEBUG nova.compute.utils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 627.093103] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg beef60d2e44f4ab480d8581e454abfb5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 627.099105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 7c0e0eabb7d44e469ee93136ce6352eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 627.099105] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 627.099105] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 627.102774] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c0e0eabb7d44e469ee93136ce6352eb [ 627.103363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg beef60d2e44f4ab480d8581e454abfb5 [ 627.195094] env[62736]: DEBUG nova.policy [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b899edcd821446b3ae6bc996b6a26037', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4346f2d2591d4970bf2f06626607e293', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 627.454900] env[62736]: INFO nova.scheduler.client.report [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Deleted allocations for instance c7670280-b5a7-4de2-8cf9-6814a81eb1a6 [ 627.461683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Expecting reply to msg 3284352a98bc4faf9f33872a6dbc79cd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 627.475457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3284352a98bc4faf9f33872a6dbc79cd [ 627.596387] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 627.598191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg ca4eb853c1384f1db81b2e627d8d2d16 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 627.639063] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca4eb853c1384f1db81b2e627d8d2d16 [ 627.963591] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eec3f13e-5e2e-472a-bf7d-0b17670f6a28 tempest-ServersTestManualDisk-1672934667 tempest-ServersTestManualDisk-1672934667-project-member] Lock "c7670280-b5a7-4de2-8cf9-6814a81eb1a6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 71.633s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.964191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg aa9855362664464793f46a89956c7e14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 627.979539] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa9855362664464793f46a89956c7e14 [ 627.995367] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1776a8eb-a1fb-4d64-af69-8e8d32ca05ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.004344] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f7862d9-e628-402a-8731-855459062efe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.038600] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6b3d788-14dc-4596-a56a-a487de51c28b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.047328] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfe42b38-08d7-44b8-b6be-06b6892d92f4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.062632] env[62736]: DEBUG nova.compute.provider_tree [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 628.063150] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 35ad7199baf341a6a8e37a5bee723fcc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 628.071231] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35ad7199baf341a6a8e37a5bee723fcc [ 628.105312] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 6e8e684948d94c9aa04d2f419053d6de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 628.127166] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Successfully created port: 0b41917f-afed-4e53-b215-68a6e91bbd7d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 628.155545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e8e684948d94c9aa04d2f419053d6de [ 628.470080] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 628.471402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg a322779afd554342a32b562172c0c111 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 628.535029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a322779afd554342a32b562172c0c111 [ 628.569046] env[62736]: DEBUG nova.scheduler.client.report [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 628.571631] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 2ca62eac83d647eea225f28a30cc9831 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 628.586472] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ca62eac83d647eea225f28a30cc9831 [ 628.627295] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 628.646049] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 628.646188] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 628.646307] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 628.646485] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 628.646625] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 628.646767] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 628.646964] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 628.647117] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 628.647297] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 628.647455] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 628.647619] env[62736]: DEBUG nova.virt.hardware [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 628.648501] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2982948-7e40-4a59-ab58-3a7d057af4c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.657047] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff320b8e-421a-4eff-b7cc-4879db451346 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 628.929377] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "d633de23-23ab-4297-b2d6-2702906b31a9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.929816] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "d633de23-23ab-4297-b2d6-2702906b31a9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.970504] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.970808] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.993119] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.075261] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.490s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.075873] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 629.077633] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg fb45eff5fb344783911b0449fd797e47 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 629.078651] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.177s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.080212] env[62736]: INFO nova.compute.claims [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 629.081575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 91aa8488e61448bfba6cd4e42938311f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 629.129691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91aa8488e61448bfba6cd4e42938311f [ 629.133246] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb45eff5fb344783911b0449fd797e47 [ 629.584618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 47645caa21de488eaf4b74072ed5a69f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 629.586280] env[62736]: DEBUG nova.compute.utils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 629.587175] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg f12b265f6f984816838ebfd0d05c078d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 629.588119] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 629.588290] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 629.595717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47645caa21de488eaf4b74072ed5a69f [ 629.604213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f12b265f6f984816838ebfd0d05c078d [ 629.645809] env[62736]: DEBUG nova.policy [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '87f04a89778e4350a90bcc998a96de0f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '77a894bddd55484d956e0569e6d26d38', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 630.092395] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 630.094026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 4cb7983bf5d24021a37ae19a4ca0339f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 630.133952] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Successfully created port: f755d6f1-e7ae-4c6f-9417-ff0e25adb144 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 630.148953] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4cb7983bf5d24021a37ae19a4ca0339f [ 630.229220] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 630.229557] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 630.230242] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 8f51e4d5983b4737af8ba5b6d072ec66 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 630.245491] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f51e4d5983b4737af8ba5b6d072ec66 [ 630.520037] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74f0f473-9421-4ef7-ad4d-f59e1725defb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.527896] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e26f4872-7de7-47ed-bf5e-d6818eba4279 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.565340] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d5118ff-4066-4917-a589-2b364b0d1d16 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.575784] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b1ae756-ce48-4a31-9920-eb9809fb73c3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 630.590530] env[62736]: DEBUG nova.compute.provider_tree [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 630.591068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg be9d57014cef459a9c43ae7589eba819 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 630.602178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 119cbea88e0143ce8e5938192d91d06f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 630.602375] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be9d57014cef459a9c43ae7589eba819 [ 630.641934] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 119cbea88e0143ce8e5938192d91d06f [ 630.737609] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 630.737609] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 630.737609] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 630.737609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 0162f4ab0d614fb790d9b8c49bec8942 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 630.753567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0162f4ab0d614fb790d9b8c49bec8942 [ 631.096995] env[62736]: DEBUG nova.scheduler.client.report [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 631.096995] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 356f49af777b445db7ebce55919dd899 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 631.106813] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 631.115601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 356f49af777b445db7ebce55919dd899 [ 631.140202] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 631.140540] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 631.141201] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 631.141464] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 631.141678] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 631.142101] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 631.142101] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 631.142319] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 631.142582] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 631.142897] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 631.143144] env[62736]: DEBUG nova.virt.hardware [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 631.144094] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3976e383-a3e3-4c5f-b077-db933f88d69c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.154044] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fc2d0c6-0bcf-4d27-a7cf-957bea9ebada {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 631.240432] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 631.240622] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 631.240707] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 631.240829] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 631.240955] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 631.241157] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.241309] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.241449] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.241595] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.241733] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.241877] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.242000] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 631.242248] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 631.242590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg d3d5e71e5b2b42179621a559f5b12c3d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 631.252576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3d5e71e5b2b42179621a559f5b12c3d [ 631.599140] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.520s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 631.599686] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 631.601322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 5b9c542793754e8a9d34c1f99d9c6871 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 631.602333] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.673s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.603728] env[62736]: INFO nova.compute.claims [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 631.605339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg cafe7b4993e14d69a5880986ba2b4ec7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 631.651591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b9c542793754e8a9d34c1f99d9c6871 [ 631.656124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cafe7b4993e14d69a5880986ba2b4ec7 [ 631.688183] env[62736]: ERROR nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 631.688183] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 631.688183] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 631.688183] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 631.688183] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 631.688183] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 631.688183] env[62736]: ERROR nova.compute.manager raise self.value [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 631.688183] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 631.688183] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 631.688183] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 631.688599] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 631.688599] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 631.688599] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 631.688599] env[62736]: ERROR nova.compute.manager [ 631.688599] env[62736]: Traceback (most recent call last): [ 631.688599] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 631.688599] env[62736]: listener.cb(fileno) [ 631.688599] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 631.688599] env[62736]: result = function(*args, **kwargs) [ 631.688599] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 631.688599] env[62736]: return func(*args, **kwargs) [ 631.688599] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 631.688599] env[62736]: raise e [ 631.688599] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 631.688599] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 631.688599] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 631.688599] env[62736]: created_port_ids = self._update_ports_for_instance( [ 631.688599] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 631.688599] env[62736]: with excutils.save_and_reraise_exception(): [ 631.688599] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 631.688599] env[62736]: self.force_reraise() [ 631.688599] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 631.688599] env[62736]: raise self.value [ 631.688599] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 631.688599] env[62736]: updated_port = self._update_port( [ 631.688599] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 631.688599] env[62736]: _ensure_no_port_binding_failure(port) [ 631.688599] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 631.688599] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 631.689236] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 631.689236] env[62736]: Removing descriptor: 17 [ 631.689236] env[62736]: ERROR nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Traceback (most recent call last): [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] yield resources [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.driver.spawn(context, instance, image_meta, [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 631.689236] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] vm_ref = self.build_virtual_machine(instance, [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] vif_infos = vmwarevif.get_vif_info(self._session, [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] for vif in network_info: [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self._sync_wrapper(fn, *args, **kwargs) [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.wait() [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self[:] = self._gt.wait() [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self._exit_event.wait() [ 631.689532] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] result = hub.switch() [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self.greenlet.switch() [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] result = function(*args, **kwargs) [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return func(*args, **kwargs) [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise e [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] nwinfo = self.network_api.allocate_for_instance( [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 631.689816] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] created_port_ids = self._update_ports_for_instance( [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] with excutils.save_and_reraise_exception(): [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.force_reraise() [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise self.value [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] updated_port = self._update_port( [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] _ensure_no_port_binding_failure(port) [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 631.690853] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise exception.PortBindingFailed(port_id=port['id']) [ 631.691127] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 631.691127] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] [ 631.691127] env[62736]: INFO nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Terminating instance [ 631.692522] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquiring lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 631.692522] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquired lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 631.692622] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 631.693029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 6a290e46ca29483d922a47a524810c06 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 631.704295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a290e46ca29483d922a47a524810c06 [ 631.717349] env[62736]: DEBUG nova.compute.manager [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Received event network-changed-0b41917f-afed-4e53-b215-68a6e91bbd7d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 631.717644] env[62736]: DEBUG nova.compute.manager [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Refreshing instance network info cache due to event network-changed-0b41917f-afed-4e53-b215-68a6e91bbd7d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 631.717853] env[62736]: DEBUG oslo_concurrency.lockutils [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] Acquiring lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 631.744959] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.108777] env[62736]: DEBUG nova.compute.utils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 632.109441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 897d4364c650496b9e03c0aa31082f59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.111574] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 93a1cb315953468881bbc99aa63f5c81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.114915] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 632.122527] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 897d4364c650496b9e03c0aa31082f59 [ 632.126494] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93a1cb315953468881bbc99aa63f5c81 [ 632.209003] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 632.279970] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 632.282207] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 46cef82597fa42598bc2736ce985cb82 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.282975] env[62736]: DEBUG nova.compute.manager [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Received event network-changed-f755d6f1-e7ae-4c6f-9417-ff0e25adb144 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 632.283290] env[62736]: DEBUG nova.compute.manager [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Refreshing instance network info cache due to event network-changed-f755d6f1-e7ae-4c6f-9417-ff0e25adb144. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 632.283353] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] Acquiring lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 632.283517] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] Acquired lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 632.283680] env[62736]: DEBUG nova.network.neutron [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Refreshing network info cache for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 632.284059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] Expecting reply to msg 42bcea63207444aa94b57937a45b6802 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.289278] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46cef82597fa42598bc2736ce985cb82 [ 632.292090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42bcea63207444aa94b57937a45b6802 [ 632.296216] env[62736]: ERROR nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 632.296216] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 632.296216] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 632.296216] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 632.296216] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 632.296216] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 632.296216] env[62736]: ERROR nova.compute.manager raise self.value [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 632.296216] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 632.296216] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 632.296216] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 632.296607] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 632.296607] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 632.296607] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 632.296607] env[62736]: ERROR nova.compute.manager [ 632.296607] env[62736]: Traceback (most recent call last): [ 632.296607] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 632.296607] env[62736]: listener.cb(fileno) [ 632.296607] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 632.296607] env[62736]: result = function(*args, **kwargs) [ 632.296607] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 632.296607] env[62736]: return func(*args, **kwargs) [ 632.296607] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 632.296607] env[62736]: raise e [ 632.296607] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 632.296607] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 632.296607] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 632.296607] env[62736]: created_port_ids = self._update_ports_for_instance( [ 632.296607] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 632.296607] env[62736]: with excutils.save_and_reraise_exception(): [ 632.296607] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 632.296607] env[62736]: self.force_reraise() [ 632.296607] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 632.296607] env[62736]: raise self.value [ 632.296607] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 632.296607] env[62736]: updated_port = self._update_port( [ 632.296607] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 632.296607] env[62736]: _ensure_no_port_binding_failure(port) [ 632.296607] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 632.296607] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 632.297241] env[62736]: nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 632.297241] env[62736]: Removing descriptor: 16 [ 632.297241] env[62736]: ERROR nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Traceback (most recent call last): [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] yield resources [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.driver.spawn(context, instance, image_meta, [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 632.297241] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] vm_ref = self.build_virtual_machine(instance, [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] vif_infos = vmwarevif.get_vif_info(self._session, [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] for vif in network_info: [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self._sync_wrapper(fn, *args, **kwargs) [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.wait() [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self[:] = self._gt.wait() [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self._exit_event.wait() [ 632.297562] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] result = hub.switch() [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self.greenlet.switch() [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] result = function(*args, **kwargs) [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return func(*args, **kwargs) [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise e [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] nwinfo = self.network_api.allocate_for_instance( [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 632.297848] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] created_port_ids = self._update_ports_for_instance( [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] with excutils.save_and_reraise_exception(): [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.force_reraise() [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise self.value [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] updated_port = self._update_port( [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] _ensure_no_port_binding_failure(port) [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 632.298141] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise exception.PortBindingFailed(port_id=port['id']) [ 632.298413] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 632.298413] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] [ 632.298413] env[62736]: INFO nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Terminating instance [ 632.300232] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 632.615915] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 632.617840] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg b3cebf5706dd4d6a8921bbe90d3a4bde in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.678677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3cebf5706dd4d6a8921bbe90d3a4bde [ 632.786858] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Releasing lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 632.787341] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 632.787529] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 632.789895] env[62736]: DEBUG oslo_concurrency.lockutils [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] Acquired lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 632.790080] env[62736]: DEBUG nova.network.neutron [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Refreshing network info cache for port 0b41917f-afed-4e53-b215-68a6e91bbd7d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 632.791251] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] Expecting reply to msg 2b29edda0fa74af1a86f07ee02a9f7ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.791504] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-05630d13-ee67-4d25-8212-7c577ea545b0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.808446] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b4fa9bf-8824-4188-bdb7-cb2abba2b3d2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 632.821786] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b29edda0fa74af1a86f07ee02a9f7ee [ 632.830034] env[62736]: DEBUG nova.network.neutron [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 632.838251] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a3182b4a-1846-4766-8988-aab0a74fa4c8 could not be found. [ 632.838617] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 632.838918] env[62736]: INFO nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Took 0.05 seconds to destroy the instance on the hypervisor. [ 632.839292] env[62736]: DEBUG oslo.service.loopingcall [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 632.842214] env[62736]: DEBUG nova.compute.manager [-] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 632.842426] env[62736]: DEBUG nova.network.neutron [-] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 632.865958] env[62736]: DEBUG nova.network.neutron [-] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 632.866495] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4aefab07b1f040dc94a1b07abc195677 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.874989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4aefab07b1f040dc94a1b07abc195677 [ 632.969568] env[62736]: DEBUG nova.network.neutron [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 632.970099] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] Expecting reply to msg e516ca9923a244bdba068ced705968e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 632.981492] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e516ca9923a244bdba068ced705968e8 [ 633.075999] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8566e938-3763-43b9-a868-49d2685737ed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.084116] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05eccfc6-6c9b-4f02-bd14-b031f83ff0ef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.123063] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45dec5df-6edd-46cc-8fe3-9cb67fb36099 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.125145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg da419ed070e543fba3721b1f7efdd485 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.140241] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef7ac011-142f-4c85-b002-9c5825f5df36 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.153640] env[62736]: DEBUG nova.compute.provider_tree [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 633.154149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg ed39d8c75a3d4c65ada35305fb944c8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.164262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed39d8c75a3d4c65ada35305fb944c8b [ 633.165239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da419ed070e543fba3721b1f7efdd485 [ 633.192063] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 633.192291] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 633.316928] env[62736]: DEBUG nova.network.neutron [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 633.369733] env[62736]: DEBUG nova.network.neutron [-] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 633.369733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5bc4d7a8b39a446984135c69633c581a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.377417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bc4d7a8b39a446984135c69633c581a [ 633.384339] env[62736]: DEBUG nova.network.neutron [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 633.384339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] Expecting reply to msg 2c9ae4390cd04e919011753539792a60 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.395109] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c9ae4390cd04e919011753539792a60 [ 633.476036] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4a716b4-847c-40e8-b1d5-c263125b9e56 req-2d10ae9f-0aae-4c4a-bfb0-e7c7e3794895 service nova] Releasing lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 633.476036] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquired lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 633.476036] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 633.476036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg abb8d6c37f214942bf7ad794de3d0fc2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.485267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abb8d6c37f214942bf7ad794de3d0fc2 [ 633.639439] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 633.660376] env[62736]: DEBUG nova.scheduler.client.report [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 633.661604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 2c6c9770a5ba4aa585aa0a9b84086af2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 633.671478] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 633.671735] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 633.671898] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 633.672093] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 633.672256] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 633.672412] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 633.672871] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 633.672871] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 633.672964] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 633.673120] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 633.673297] env[62736]: DEBUG nova.virt.hardware [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 633.674164] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56d70cbb-4af5-4869-b7a8-3f446f1ea2e9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.681342] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c6c9770a5ba4aa585aa0a9b84086af2 [ 633.684221] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b911972-7b5b-4194-8293-e8439d206932 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.698653] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 633.705514] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating folder: Project (09e87409ac5e452896b8b112d0fd8722). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 633.706104] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b21c9143-b8d0-4ed7-842e-aec9a046143a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.718873] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created folder: Project (09e87409ac5e452896b8b112d0fd8722) in parent group-v107913. [ 633.718873] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating folder: Instances. Parent ref: group-v107921. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 633.719129] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a23dfb93-95e8-4457-bd69-91fb62a617b6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.736494] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created folder: Instances in parent group-v107921. [ 633.736494] env[62736]: DEBUG oslo.service.loopingcall [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 633.736764] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 633.736906] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-fd4aa7bc-4b9f-42fb-b6b2-6f424020a3e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 633.754193] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 633.754193] env[62736]: value = "task-397682" [ 633.754193] env[62736]: _type = "Task" [ 633.754193] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 633.762654] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397682, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 633.871584] env[62736]: INFO nova.compute.manager [-] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Took 1.03 seconds to deallocate network for instance. [ 633.873523] env[62736]: DEBUG nova.compute.claims [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 633.873708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 633.885427] env[62736]: DEBUG oslo_concurrency.lockutils [req-429d65cd-fb53-42a8-8f1d-89e0f530575e req-b815c065-9e1a-47d3-a717-b1fe34a891d5 service nova] Releasing lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 633.944254] env[62736]: DEBUG nova.compute.manager [req-463627ff-756d-4758-9ea5-4d4d284c875f req-ac3f2b2e-f32e-49ef-b430-f525f1824be6 service nova] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Received event network-vif-deleted-0b41917f-afed-4e53-b215-68a6e91bbd7d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 634.011997] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 634.093179] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 634.093511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg dad883d30c4246c0a13cef5c50526cee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.102434] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dad883d30c4246c0a13cef5c50526cee [ 634.165554] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.563s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.165940] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 634.167732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg bcc03e9a98374e13aba70d2a8089efb1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.178517] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.175s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.178517] env[62736]: INFO nova.compute.claims [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 634.178517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 26733cefa3134733b61acb5d7196bd68 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.207584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bcc03e9a98374e13aba70d2a8089efb1 [ 634.234011] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26733cefa3134733b61acb5d7196bd68 [ 634.264147] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397682, 'name': CreateVM_Task, 'duration_secs': 0.312953} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 634.264147] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 634.264514] env[62736]: DEBUG oslo_vmware.service [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4dd5cbf2-e5c8-4b0f-bb79-dad51e14a712 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.270489] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 634.270755] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 634.271344] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 634.271422] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2500458d-e245-4889-bc60-fe384069ca7d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.275796] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 634.275796] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]527e5972-9219-d35f-bf31-187bc061da41" [ 634.275796] env[62736]: _type = "Task" [ 634.275796] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 634.287887] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]527e5972-9219-d35f-bf31-187bc061da41, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 634.595837] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Releasing lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 634.596306] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 634.596489] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 634.596782] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4dc20d5a-0c04-4f32-8eaa-67e8f881e13d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.605440] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45de5500-329f-41d8-a5b9-9fe196262626 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.628824] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c97b25ac-527f-404e-a539-1c0f43bcd66b could not be found. [ 634.629360] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 634.629565] env[62736]: INFO nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Took 0.03 seconds to destroy the instance on the hypervisor. [ 634.629811] env[62736]: DEBUG oslo.service.loopingcall [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 634.630012] env[62736]: DEBUG nova.compute.manager [-] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 634.630098] env[62736]: DEBUG nova.network.neutron [-] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 634.647127] env[62736]: DEBUG nova.network.neutron [-] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 634.648055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2dfd9f5fa3ce412aa78e276c0cded1d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.656640] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dfd9f5fa3ce412aa78e276c0cded1d7 [ 634.681359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8f901bf4a46f40028e6bf2bd0a620012 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.686059] env[62736]: DEBUG nova.compute.utils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 634.686059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg da4fd3c76a75445b92768a495a204b4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 634.686059] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 634.686059] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 634.691138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f901bf4a46f40028e6bf2bd0a620012 [ 634.699023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da4fd3c76a75445b92768a495a204b4c [ 634.702918] env[62736]: DEBUG nova.compute.manager [req-14dc12d4-3394-45c7-b2d7-b160f34111a9 req-a35276c6-7828-4ffa-be09-ec8e8f1f7fc5 service nova] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Received event network-vif-deleted-f755d6f1-e7ae-4c6f-9417-ff0e25adb144 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 634.785438] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 634.785886] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 634.786240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 634.786426] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 634.786638] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 634.787360] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3947645d-5143-4a20-bdf0-8335e0f14ff2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.805619] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 634.805903] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 634.806676] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17194303-6f8b-479b-9b9f-626bdcb28726 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.813805] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7da8b932-f7e1-408f-9b75-0258964e8b08 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.820468] env[62736]: DEBUG nova.policy [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '811d5d3302b64a9ab3c07cbbcec100d4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'ec2a7b4f1aea43cfa1972b508670852d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 634.823492] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 634.823492] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52b12d10-38f7-f94c-3bdc-524c8bca992b" [ 634.823492] env[62736]: _type = "Task" [ 634.823492] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 634.832778] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52b12d10-38f7-f94c-3bdc-524c8bca992b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 635.150578] env[62736]: DEBUG nova.network.neutron [-] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 635.151081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 10c02c72e2d44197bc811337c90265ba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 635.160344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10c02c72e2d44197bc811337c90265ba [ 635.188588] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 635.190090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg e9bb0ccc5f3c49e6b3ed928cc80db119 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 635.232636] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9bb0ccc5f3c49e6b3ed928cc80db119 [ 635.337449] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Preparing fetch location {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 635.337726] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating directory with path [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 635.338143] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5a041aaf-a924-486b-be0b-4813f0d3f00b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.364894] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created directory with path [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 635.365093] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Fetch image to [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 635.365377] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Downloading image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk on the data store datastore2 {{(pid=62736) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 635.366579] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4ff22ba-7dc2-43e1-8d6a-0d7e28b40de7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.385918] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c53010ac-bc28-4ea0-8d2e-1a2a8cc40ea6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.398308] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b01de939-e27c-41ef-ab64-3dcd7f8f0bb7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.441338] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-089c5adb-9fde-43bb-a108-0d049f181246 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.449741] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Successfully created port: 48e4b255-39f8-4691-be1e-141dd9425261 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 635.455706] env[62736]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b111bd66-0823-4c8b-bc8a-d11cb0bec4c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.540909] env[62736]: DEBUG nova.virt.vmwareapi.images [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Downloading image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to the data store datastore2 {{(pid=62736) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 635.610050] env[62736]: DEBUG oslo_vmware.rw_handles [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62736) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 635.672787] env[62736]: INFO nova.compute.manager [-] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Took 1.04 seconds to deallocate network for instance. [ 635.679447] env[62736]: DEBUG nova.compute.claims [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 635.679653] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 635.698314] env[62736]: INFO nova.virt.block_device [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Booting with volume e84143f7-236e-492d-9d69-0610b37fb002 at /dev/sda [ 635.773684] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a86a213e-03d0-4585-ae60-1f3b5578e860 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.786571] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fee243b-6b85-4a10-b7d0-16b9ab2ecb3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.838176] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2d19a601-a283-4a5d-81ae-beedc17dba79 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.845523] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7f4681a-3b00-4a2c-82d3-1463525ff717 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.858293] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bd949c8-1096-4c77-aa09-10a2379dfc95 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.882195] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3589db6-59ce-44f8-8679-a7554e174997 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.894744] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f76ad096-733c-43fc-9b21-5cca0080db6d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.927548] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-689b0d95-b0ad-44aa-9196-73e3bb2f9db1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.939336] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b93dc2a0-349a-480a-b36b-b0bf38ed0231 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.947612] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0d62c3a-9e59-4f1d-a605-dca224db8281 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.957154] env[62736]: DEBUG nova.virt.block_device [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating existing volume attachment record: f4230336-3c40-4d38-b1a5-e44f521d0e02 {{(pid=62736) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 635.972649] env[62736]: DEBUG nova.compute.provider_tree [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 635.973558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg e651a2ddfafa4f5199a929ebd302e137 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 635.981754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e651a2ddfafa4f5199a929ebd302e137 [ 636.360163] env[62736]: DEBUG oslo_vmware.rw_handles [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Completed reading data from the image iterator. {{(pid=62736) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 636.360451] env[62736]: DEBUG oslo_vmware.rw_handles [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=62736) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 636.418765] env[62736]: DEBUG nova.virt.vmwareapi.images [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Downloaded image file data 81867c62-ef8e-483f-bfd2-854abdcd6db5 to vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk on the data store datastore2 {{(pid=62736) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 636.421047] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Caching image {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 636.421310] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copying Virtual Disk [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk to [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 636.421586] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-69eecd1e-a2d0-4f06-afc5-8ee701a097cb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.433721] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 636.433721] env[62736]: value = "task-397683" [ 636.433721] env[62736]: _type = "Task" [ 636.433721] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 636.443867] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397683, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 636.479121] env[62736]: DEBUG nova.scheduler.client.report [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 636.482737] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 047ca5e725354e74936df95040dacc3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 636.498193] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 047ca5e725354e74936df95040dacc3a [ 636.522341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 82d0b63ed3434daca90253add20ad3ba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 636.534771] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82d0b63ed3434daca90253add20ad3ba [ 636.950890] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397683, 'name': CopyVirtualDisk_Task} progress is 100%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 636.984970] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.816s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.985481] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 636.987218] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 43efd95d76134f0592f02fd4e3da0811 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 636.988316] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.747s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.996323] env[62736]: INFO nova.compute.claims [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 636.998082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg a24a362279854a16a7ab413f970a8d49 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.000075] env[62736]: ERROR nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 637.000075] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 637.000075] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 637.000075] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 637.000075] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 637.000075] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 637.000075] env[62736]: ERROR nova.compute.manager raise self.value [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 637.000075] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 637.000075] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 637.000075] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 637.000532] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 637.000532] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 637.000532] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 637.000532] env[62736]: ERROR nova.compute.manager [ 637.000532] env[62736]: Traceback (most recent call last): [ 637.000532] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 637.000532] env[62736]: listener.cb(fileno) [ 637.000532] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 637.000532] env[62736]: result = function(*args, **kwargs) [ 637.000532] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 637.000532] env[62736]: return func(*args, **kwargs) [ 637.000532] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 637.000532] env[62736]: raise e [ 637.000532] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 637.000532] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 637.000532] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 637.000532] env[62736]: created_port_ids = self._update_ports_for_instance( [ 637.000532] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 637.000532] env[62736]: with excutils.save_and_reraise_exception(): [ 637.000532] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 637.000532] env[62736]: self.force_reraise() [ 637.000532] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 637.000532] env[62736]: raise self.value [ 637.000532] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 637.000532] env[62736]: updated_port = self._update_port( [ 637.000532] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 637.000532] env[62736]: _ensure_no_port_binding_failure(port) [ 637.000532] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 637.000532] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 637.001534] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 637.001534] env[62736]: Removing descriptor: 16 [ 637.039177] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43efd95d76134f0592f02fd4e3da0811 [ 637.057422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a24a362279854a16a7ab413f970a8d49 [ 637.080088] env[62736]: DEBUG nova.compute.manager [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Received event network-changed-48e4b255-39f8-4691-be1e-141dd9425261 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 637.080088] env[62736]: DEBUG nova.compute.manager [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Refreshing instance network info cache due to event network-changed-48e4b255-39f8-4691-be1e-141dd9425261. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 637.080088] env[62736]: DEBUG oslo_concurrency.lockutils [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] Acquiring lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 637.080088] env[62736]: DEBUG oslo_concurrency.lockutils [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] Acquired lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 637.080088] env[62736]: DEBUG nova.network.neutron [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Refreshing network info cache for port 48e4b255-39f8-4691-be1e-141dd9425261 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 637.080406] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] Expecting reply to msg 4e3d5de1bf3b4c6bb9e2dd5e6221f9a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.082598] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 3bc8bf5b7073400a842585c6cdab1f0b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.091181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e3d5de1bf3b4c6bb9e2dd5e6221f9a0 [ 637.092742] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3bc8bf5b7073400a842585c6cdab1f0b [ 637.447801] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397683, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.680543} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 637.448279] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copied Virtual Disk [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk to [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 637.448497] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleting the datastore file [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5/tmp-sparse.vmdk {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 637.448762] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-71841519-741a-4176-ba25-f6a071370a72 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.455773] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 637.455773] env[62736]: value = "task-397684" [ 637.455773] env[62736]: _type = "Task" [ 637.455773] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 637.464941] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397684, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 637.502441] env[62736]: DEBUG nova.compute.utils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 637.503104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg e7dfaede03514e69884bef05497b19c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.509557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg f3b4588b42c94dc6acfd3eb4d5af2f39 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.512041] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 637.517155] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3b4588b42c94dc6acfd3eb4d5af2f39 [ 637.521210] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7dfaede03514e69884bef05497b19c6 [ 637.589527] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 600bc1c0a3d2467c84bfe9e659793a2a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.633148] env[62736]: DEBUG nova.network.neutron [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 637.635434] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 600bc1c0a3d2467c84bfe9e659793a2a [ 637.854639] env[62736]: DEBUG nova.network.neutron [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 637.855174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] Expecting reply to msg 43d461c09f3d4cc99a0d4ce85659645e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 637.872180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43d461c09f3d4cc99a0d4ce85659645e [ 637.971708] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397684, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.022027} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 637.971708] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 637.971708] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Moving file from [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9/81867c62-ef8e-483f-bfd2-854abdcd6db5 to [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5. {{(pid=62736) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:334}} [ 637.971708] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MoveDatastoreFile_Task with opID=oslo.vmware-7ad93876-2ec7-4b26-86a5-0c319183fd4f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.981936] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 637.981936] env[62736]: value = "task-397685" [ 637.981936] env[62736]: _type = "Task" [ 637.981936] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 637.994331] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397685, 'name': MoveDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 638.011280] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 638.013568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 13579e0d46e34f25acf1d59d3b1b792a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 638.064464] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13579e0d46e34f25acf1d59d3b1b792a [ 638.101914] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 638.102439] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 638.102696] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 638.102890] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 638.103164] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 638.103320] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 638.103504] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 638.103773] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 638.103996] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 638.104348] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 638.104574] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 638.104760] env[62736]: DEBUG nova.virt.hardware [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 638.105786] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da6ab1c9-f6c4-4df9-a669-b36a0afa0f91 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.114733] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ad966ef-04c9-454a-92e8-1fb4e1d161ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.136184] env[62736]: ERROR nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Traceback (most recent call last): [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] yield resources [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.driver.spawn(context, instance, image_meta, [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self._vmops.spawn(context, instance, image_meta, injected_files, [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] vm_ref = self.build_virtual_machine(instance, [ 638.136184] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] vif_infos = vmwarevif.get_vif_info(self._session, [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] for vif in network_info: [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return self._sync_wrapper(fn, *args, **kwargs) [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.wait() [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self[:] = self._gt.wait() [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return self._exit_event.wait() [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 638.136521] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] current.throw(*self._exc) [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] result = function(*args, **kwargs) [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return func(*args, **kwargs) [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise e [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] nwinfo = self.network_api.allocate_for_instance( [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] created_port_ids = self._update_ports_for_instance( [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] with excutils.save_and_reraise_exception(): [ 638.136901] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.force_reraise() [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise self.value [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] updated_port = self._update_port( [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] _ensure_no_port_binding_failure(port) [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise exception.PortBindingFailed(port_id=port['id']) [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 638.137572] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] [ 638.137572] env[62736]: INFO nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Terminating instance [ 638.140907] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquiring lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 638.358821] env[62736]: DEBUG oslo_concurrency.lockutils [req-d61ce7bf-2183-4f44-83e3-5bb23f53d8c3 req-8f980e99-2ec6-406d-aa29-997a7dba7bdb service nova] Releasing lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 638.359236] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquired lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 638.359444] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 638.359895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 9f428412542b45bc9dd371e26c143ffc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 638.371944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f428412542b45bc9dd371e26c143ffc [ 638.487535] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397685, 'name': MoveDatastoreFile_Task, 'duration_secs': 0.025278} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 638.487797] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] File moved {{(pid=62736) file_move /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:346}} [ 638.487964] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Cleaning up location [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:640}} [ 638.488131] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleting the datastore file [datastore2] vmware_temp/6b094e57-e4c0-4b1e-b65a-3fbf041e5dc9 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 638.488370] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-dee9ca40-4955-4fe9-893f-cc28a9688a8a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.491891] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8faf442-3daa-42a8-ba3a-bd4163cc63c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.495540] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 638.495540] env[62736]: value = "task-397686" [ 638.495540] env[62736]: _type = "Task" [ 638.495540] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 638.506917] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2f60711-4290-4f08-8c8e-0186af9ce921 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.516923] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397686, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 638.549760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8b3313093e02449da5b8963fa0be6396 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 638.551499] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1a86e96-5520-40fe-8f5e-c1c8f8328c55 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.569499] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8a767aa3-103d-4c98-919c-2e3dba55a685 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.589746] env[62736]: DEBUG nova.compute.provider_tree [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 638.590258] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 7078a7e5215c41edaf5a14d058bcb5cc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 638.593949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b3313093e02449da5b8963fa0be6396 [ 638.599282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7078a7e5215c41edaf5a14d058bcb5cc [ 639.006454] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397686, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.024146} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 639.006707] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 639.007402] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2ed78215-3d56-4e2f-8ffb-07556e63cde0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.012510] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 639.012510] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]525d9ef7-4c84-6f10-1405-d92e3146d725" [ 639.012510] env[62736]: _type = "Task" [ 639.012510] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.021504] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525d9ef7-4c84-6f10-1405-d92e3146d725, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.056505] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 639.064358] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 639.085836] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 639.086111] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 639.086269] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 639.086466] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 639.086614] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 639.086755] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 639.086952] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 639.087102] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 639.087261] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 639.087447] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 639.087622] env[62736]: DEBUG nova.virt.hardware [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 639.088550] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af798d65-9bc1-462e-bd02-a845303843ee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.097024] env[62736]: DEBUG nova.scheduler.client.report [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 639.100889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 26659908ff2746e69bd05d1e0bb20e18 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 639.101481] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-493e218c-90e4-4bcc-8374-c758e9033c4a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.118396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26659908ff2746e69bd05d1e0bb20e18 [ 639.119053] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 639.124817] env[62736]: DEBUG oslo.service.loopingcall [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 639.125354] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 639.125564] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-5a7a4fb5-4c76-4c15-84f3-eccbe23643da {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.149986] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 639.149986] env[62736]: value = "task-397687" [ 639.149986] env[62736]: _type = "Task" [ 639.149986] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.159803] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397687, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.240099] env[62736]: DEBUG nova.compute.manager [req-37a29b96-3d5a-46cc-92c5-486e37d20c8e req-f2c94581-e925-4b65-bcb5-5edb5c45e6d5 service nova] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Received event network-vif-deleted-48e4b255-39f8-4691-be1e-141dd9425261 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 639.469790] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 639.469790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 260885c1ed9e40ef8048a863612995d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 639.478090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 260885c1ed9e40ef8048a863612995d1 [ 639.522680] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525d9ef7-4c84-6f10-1405-d92e3146d725, 'name': SearchDatastore_Task, 'duration_secs': 0.008176} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 639.523489] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.523911] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] b42fe7a4-ec30-441f-b5dc-1f705ee5c881/b42fe7a4-ec30-441f-b5dc-1f705ee5c881.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 639.524369] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d5263cd9-10f1-4947-913d-82e08f40816d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.530779] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 639.530779] env[62736]: value = "task-397688" [ 639.530779] env[62736]: _type = "Task" [ 639.530779] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 639.548238] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397688, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.607824] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.619s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.608701] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 639.611228] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg e396770ef5c047fe912573e15d9c6503 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 639.612483] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.883s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.614222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg ac823f65e090483892303d44abd965e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 639.660214] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397687, 'name': CreateVM_Task} progress is 99%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 639.672403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac823f65e090483892303d44abd965e8 [ 639.673582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e396770ef5c047fe912573e15d9c6503 [ 639.971508] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Releasing lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 639.972303] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 639.972467] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8838442a-64ca-497d-beef-64276f66f843 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 639.981682] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e480d569-277d-4c97-a423-cdb7363e0cb6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.005072] env[62736]: WARNING nova.virt.vmwareapi.driver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 4e2f1fc0-102f-4d39-bec2-a2651918a202 could not be found. [ 640.005312] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 640.005633] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5299c4a4-662b-4e74-8570-fb41f5861132 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.013434] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecae84cb-3258-4944-beae-bfbf5932ae30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.043032] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4e2f1fc0-102f-4d39-bec2-a2651918a202 could not be found. [ 640.043298] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 640.043481] env[62736]: INFO nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Took 0.07 seconds to destroy the instance on the hypervisor. [ 640.043800] env[62736]: DEBUG oslo.service.loopingcall [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 640.048721] env[62736]: DEBUG nova.compute.manager [-] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 640.048954] env[62736]: DEBUG nova.network.neutron [-] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 640.057049] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397688, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.453632} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.057049] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] b42fe7a4-ec30-441f-b5dc-1f705ee5c881/b42fe7a4-ec30-441f-b5dc-1f705ee5c881.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 640.057049] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 640.057255] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-18c2c017-1ec3-4f86-9816-a2c913c41238 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.064309] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 640.064309] env[62736]: value = "task-397689" [ 640.064309] env[62736]: _type = "Task" [ 640.064309] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.076900] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397689, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 640.085147] env[62736]: DEBUG nova.network.neutron [-] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 640.085883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ec052a47376f4d608b4cceefa6d320f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 640.093530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec052a47376f4d608b4cceefa6d320f9 [ 640.121234] env[62736]: DEBUG nova.compute.utils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 640.121872] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 95d76b39d2714ced85a31dac312df20c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 640.123773] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 640.124183] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 640.132280] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95d76b39d2714ced85a31dac312df20c [ 640.174191] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397687, 'name': CreateVM_Task, 'duration_secs': 0.531469} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.174358] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 640.174763] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 640.174908] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 640.175339] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 640.175596] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7b49ce46-070e-43f0-aa66-edbafcfac101 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.183142] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 640.183142] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5243ad3a-d8f1-cdda-47b2-d0934f0c544a" [ 640.183142] env[62736]: _type = "Task" [ 640.183142] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.191846] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5243ad3a-d8f1-cdda-47b2-d0934f0c544a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 640.296953] env[62736]: DEBUG nova.policy [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '213b05b97a734701ba942161e3b049e4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '14d7373c9b3445f3a36c0039ebd7e260', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 640.578630] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397689, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.060278} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.578965] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 640.579959] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c241c9a-0d10-4459-833a-7c763b2b75fd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.598324] env[62736]: DEBUG nova.network.neutron [-] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 640.598808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c9792a9fcce4467db6b27f69db05c14a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 640.609599] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Reconfiguring VM instance instance-00000018 to attach disk [datastore2] b42fe7a4-ec30-441f-b5dc-1f705ee5c881/b42fe7a4-ec30-441f-b5dc-1f705ee5c881.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 640.610742] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9792a9fcce4467db6b27f69db05c14a [ 640.611239] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-a2af5195-c0bf-47d7-9873-b7c908b9cb17 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.627135] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 640.628839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg c61f6ab1b629451a9e8d39038b6ee215 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 640.630304] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-132c246f-ca20-4d92-94a8-b78f4289ffaf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.644815] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 640.644815] env[62736]: value = "task-397690" [ 640.644815] env[62736]: _type = "Task" [ 640.644815] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.645885] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51097a57-6938-46ec-982c-6bc2f3de51c7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.658334] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397690, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 640.690945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c61f6ab1b629451a9e8d39038b6ee215 [ 640.695214] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f31c3cee-237d-418d-8fb5-15040ae86839 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.698428] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Successfully created port: 08f341c7-922c-41cc-8984-e8f0675c276f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 640.713579] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa67f3a4-3550-4e33-88dd-2157ec3bac42 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.724315] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5243ad3a-d8f1-cdda-47b2-d0934f0c544a, 'name': SearchDatastore_Task, 'duration_secs': 0.061166} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 640.725001] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 640.725252] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 640.725512] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 640.725623] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 640.726009] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 640.726740] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8eafcfc0-db8a-498c-bba4-e95007e78ad7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.740448] env[62736]: DEBUG nova.compute.provider_tree [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 640.740983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 323871276fb440568c5f52aa98904438 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 640.748893] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 323871276fb440568c5f52aa98904438 [ 640.751092] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 640.751092] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 640.752678] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d8192971-88aa-4461-84a6-0fe333c70aba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 640.760683] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 640.760683] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52d1b353-8e72-f10c-caa9-b133a02addee" [ 640.760683] env[62736]: _type = "Task" [ 640.760683] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 640.772340] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52d1b353-8e72-f10c-caa9-b133a02addee, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.111207] env[62736]: INFO nova.compute.manager [-] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Took 1.06 seconds to deallocate network for instance. [ 641.136529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 0ab6d91feb0a4baf849c30e75de78024 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 641.159816] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397690, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.185295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ab6d91feb0a4baf849c30e75de78024 [ 641.249920] env[62736]: DEBUG nova.scheduler.client.report [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 641.252499] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 477031ff15864f58beb2bba0bfb57f37 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 641.276597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 477031ff15864f58beb2bba0bfb57f37 [ 641.282861] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52d1b353-8e72-f10c-caa9-b133a02addee, 'name': SearchDatastore_Task, 'duration_secs': 0.01668} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 641.284253] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8fc31436-ba1d-4e26-b3d8-6e67d1e6d875 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.294892] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 641.294892] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]525f7bdf-dde0-0614-4222-4847562aa66d" [ 641.294892] env[62736]: _type = "Task" [ 641.294892] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 641.305519] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525f7bdf-dde0-0614-4222-4847562aa66d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.640187] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 641.663266] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397690, 'name': ReconfigVM_Task, 'duration_secs': 0.987965} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 641.663550] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Reconfigured VM instance instance-00000018 to attach disk [datastore2] b42fe7a4-ec30-441f-b5dc-1f705ee5c881/b42fe7a4-ec30-441f-b5dc-1f705ee5c881.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 641.664207] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-a37050c1-5b46-4ad1-8303-10053d8c5e0a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.669053] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 641.669281] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 641.669464] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 641.669657] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 641.669801] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 641.669965] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 641.670130] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 641.670465] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 641.670465] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 641.670631] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 641.670802] env[62736]: DEBUG nova.virt.hardware [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 641.671623] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc671377-9ef4-4729-b89a-753f8efd3b95 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.675357] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 641.675357] env[62736]: value = "task-397691" [ 641.675357] env[62736]: _type = "Task" [ 641.675357] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 641.683059] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32efa6db-1ec6-4b1b-94b7-4cdb35459c5a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.691292] env[62736]: INFO nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Took 0.58 seconds to detach 1 volumes for instance. [ 641.692984] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397691, 'name': Rename_Task} progress is 10%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 641.693533] env[62736]: DEBUG nova.compute.claims [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 641.693588] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 641.755521] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.143s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 641.756258] env[62736]: ERROR nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] Traceback (most recent call last): [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.driver.spawn(context, instance, image_meta, [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self._vmops.spawn(context, instance, image_meta, injected_files, [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] vm_ref = self.build_virtual_machine(instance, [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] vif_infos = vmwarevif.get_vif_info(self._session, [ 641.756258] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] for vif in network_info: [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self._sync_wrapper(fn, *args, **kwargs) [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.wait() [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self[:] = self._gt.wait() [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self._exit_event.wait() [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] result = hub.switch() [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 641.756605] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return self.greenlet.switch() [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] result = function(*args, **kwargs) [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] return func(*args, **kwargs) [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise e [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] nwinfo = self.network_api.allocate_for_instance( [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] created_port_ids = self._update_ports_for_instance( [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] with excutils.save_and_reraise_exception(): [ 641.757012] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] self.force_reraise() [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise self.value [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] updated_port = self._update_port( [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] _ensure_no_port_binding_failure(port) [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] raise exception.PortBindingFailed(port_id=port['id']) [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] nova.exception.PortBindingFailed: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. [ 641.757345] env[62736]: ERROR nova.compute.manager [instance: 558058e0-9808-415a-8866-75d1c8598d43] [ 641.757646] env[62736]: DEBUG nova.compute.utils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 641.758512] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.849s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 641.760126] env[62736]: INFO nova.compute.claims [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 641.761772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 38cbb673eb46413cb137f07291652546 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 641.763099] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Build of instance 558058e0-9808-415a-8866-75d1c8598d43 was re-scheduled: Binding failed for port fa29c075-7ce5-482f-8388-05891144179b, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 641.763567] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 641.763819] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 641.763973] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 641.764148] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 641.764590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 01a24231f5c0401398d6bb33946530c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 641.779103] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01a24231f5c0401398d6bb33946530c6 [ 641.806636] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525f7bdf-dde0-0614-4222-4847562aa66d, 'name': SearchDatastore_Task, 'duration_secs': 0.018469} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 641.806988] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 641.807388] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 641.807667] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-914095ce-46f7-46b1-8442-7d55d162c9f0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 641.818356] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 641.818356] env[62736]: value = "task-397692" [ 641.818356] env[62736]: _type = "Task" [ 641.818356] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 641.819058] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38cbb673eb46413cb137f07291652546 [ 641.830585] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397692, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.187947] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397691, 'name': Rename_Task, 'duration_secs': 0.167719} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.188507] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 642.188768] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-efcf998f-393a-43b0-91b2-a8759acf3709 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.198904] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 642.198904] env[62736]: value = "task-397693" [ 642.198904] env[62736]: _type = "Task" [ 642.198904] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.217920] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397693, 'name': PowerOnVM_Task} progress is 33%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.268316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg ace7567554e54cf8be9746730ffbb7dc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 642.278865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ace7567554e54cf8be9746730ffbb7dc [ 642.297193] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.329506] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397692, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.472016} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.329775] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 642.329978] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 642.330217] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-e40e7d5c-1ac7-4632-9724-8fea0dfa98f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.337325] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 642.337325] env[62736]: value = "task-397694" [ 642.337325] env[62736]: _type = "Task" [ 642.337325] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.345163] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397694, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.461542] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.461590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 1a5bf29f17ee419aa3127cf76a0088f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 642.473171] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a5bf29f17ee419aa3127cf76a0088f0 [ 642.518025] env[62736]: DEBUG nova.compute.manager [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Received event network-changed-08f341c7-922c-41cc-8984-e8f0675c276f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 642.518244] env[62736]: DEBUG nova.compute.manager [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Refreshing instance network info cache due to event network-changed-08f341c7-922c-41cc-8984-e8f0675c276f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 642.518949] env[62736]: DEBUG oslo_concurrency.lockutils [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] Acquiring lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.519155] env[62736]: DEBUG oslo_concurrency.lockutils [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] Acquired lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.519371] env[62736]: DEBUG nova.network.neutron [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Refreshing network info cache for port 08f341c7-922c-41cc-8984-e8f0675c276f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 642.519873] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] Expecting reply to msg 1dfa90f880a147b5b8da737bea394166 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 642.521643] env[62736]: ERROR nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 642.521643] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.521643] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.521643] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.521643] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.521643] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.521643] env[62736]: ERROR nova.compute.manager raise self.value [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.521643] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 642.521643] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.521643] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 642.522103] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.522103] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 642.522103] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 642.522103] env[62736]: ERROR nova.compute.manager [ 642.522103] env[62736]: Traceback (most recent call last): [ 642.522103] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 642.522103] env[62736]: listener.cb(fileno) [ 642.522103] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.522103] env[62736]: result = function(*args, **kwargs) [ 642.522103] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 642.522103] env[62736]: return func(*args, **kwargs) [ 642.522103] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.522103] env[62736]: raise e [ 642.522103] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.522103] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 642.522103] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.522103] env[62736]: created_port_ids = self._update_ports_for_instance( [ 642.522103] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.522103] env[62736]: with excutils.save_and_reraise_exception(): [ 642.522103] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.522103] env[62736]: self.force_reraise() [ 642.522103] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.522103] env[62736]: raise self.value [ 642.522103] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.522103] env[62736]: updated_port = self._update_port( [ 642.522103] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.522103] env[62736]: _ensure_no_port_binding_failure(port) [ 642.522103] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.522103] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 642.522848] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 642.522848] env[62736]: Removing descriptor: 16 [ 642.522848] env[62736]: ERROR nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Traceback (most recent call last): [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] yield resources [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.driver.spawn(context, instance, image_meta, [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self._vmops.spawn(context, instance, image_meta, injected_files, [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 642.522848] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] vm_ref = self.build_virtual_machine(instance, [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] vif_infos = vmwarevif.get_vif_info(self._session, [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] for vif in network_info: [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self._sync_wrapper(fn, *args, **kwargs) [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.wait() [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self[:] = self._gt.wait() [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self._exit_event.wait() [ 642.523155] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] result = hub.switch() [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self.greenlet.switch() [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] result = function(*args, **kwargs) [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return func(*args, **kwargs) [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise e [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] nwinfo = self.network_api.allocate_for_instance( [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 642.523485] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] created_port_ids = self._update_ports_for_instance( [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] with excutils.save_and_reraise_exception(): [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.force_reraise() [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise self.value [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] updated_port = self._update_port( [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] _ensure_no_port_binding_failure(port) [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 642.523825] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise exception.PortBindingFailed(port_id=port['id']) [ 642.524178] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 642.524178] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] [ 642.524178] env[62736]: INFO nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Terminating instance [ 642.525147] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquiring lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.527019] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1dfa90f880a147b5b8da737bea394166 [ 642.708885] env[62736]: DEBUG oslo_vmware.api [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397693, 'name': PowerOnVM_Task, 'duration_secs': 0.460479} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.709188] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 642.709428] env[62736]: INFO nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Took 9.07 seconds to spawn the instance on the hypervisor. [ 642.709632] env[62736]: DEBUG nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 642.710441] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccef25c0-946b-42fc-bc8d-b580a2a0d2c1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.718449] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 20b74c5c27c646daa9ee684ac9da433e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 642.770024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20b74c5c27c646daa9ee684ac9da433e [ 642.847321] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397694, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.063092} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.850068] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 642.851192] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e4aa10a-fefc-41da-bbe2-f12d7e7557a7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.873455] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Reconfiguring VM instance instance-0000001a to attach disk [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 642.881202] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-6eda1026-0e2e-4f8c-9481-743956f598e9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.908580] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 642.908580] env[62736]: value = "task-397695" [ 642.908580] env[62736]: _type = "Task" [ 642.908580] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.932166] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397695, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.955100] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.955323] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.967638] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-558058e0-9808-415a-8866-75d1c8598d43" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 642.967861] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 642.968135] env[62736]: DEBUG nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 642.968250] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 642.996474] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.997345] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 87efe764334b41f3af7e5f428672db0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.004323] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87efe764334b41f3af7e5f428672db0a [ 643.053188] env[62736]: DEBUG nova.network.neutron [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 643.155203] env[62736]: DEBUG nova.network.neutron [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.155759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] Expecting reply to msg 1771f49141d74d2fa3f50bc10dcd04da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.165224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1771f49141d74d2fa3f50bc10dcd04da [ 643.231092] env[62736]: INFO nova.compute.manager [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Took 29.34 seconds to build instance. [ 643.231477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 853edde0f84849e38046d2c6200b5597 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.245290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 853edde0f84849e38046d2c6200b5597 [ 643.269177] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e513ded0-8ba6-4294-adc4-984b5182845f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.276841] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f7a916e-4ac0-4cb7-8a27-bbb18096fd37 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.306945] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e6ea521-3278-46fe-9cec-84154a319cc7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.314322] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2947f1a5-364a-4788-8c38-8191d524efee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.327577] env[62736]: DEBUG nova.compute.provider_tree [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 643.328153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg eaf5c7d4d74f4b52be6847e82fd4380d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.337325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eaf5c7d4d74f4b52be6847e82fd4380d [ 643.437906] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397695, 'name': ReconfigVM_Task} progress is 14%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 643.499605] env[62736]: DEBUG nova.network.neutron [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 643.500214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg b7a4dc266fd947d88c080800d96a8697 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.508104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b7a4dc266fd947d88c080800d96a8697 [ 643.658631] env[62736]: DEBUG oslo_concurrency.lockutils [req-232e116d-3d22-4925-99e0-acf8731a2dd2 req-41907ee8-ffe7-440f-904b-98785248292c service nova] Releasing lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.659089] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquired lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 643.659286] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 643.659742] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg a8ef131ac91542679cbad100458be2d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.672013] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8ef131ac91542679cbad100458be2d7 [ 643.734005] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2431d4c3-8b63-4659-8b34-88717834f95b tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.764s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 643.734653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 2b7d3b6c9929414fadcf7926571bbbed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.764012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b7d3b6c9929414fadcf7926571bbbed [ 643.830702] env[62736]: DEBUG nova.scheduler.client.report [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 643.833137] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 79b3680715d84e429446a5b70889e478 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 643.846964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79b3680715d84e429446a5b70889e478 [ 643.919009] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397695, 'name': ReconfigVM_Task, 'duration_secs': 0.823186} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 643.919279] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Reconfigured VM instance instance-0000001a to attach disk [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 643.919916] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-45a7e05f-46fa-4b07-9627-8a2738d77649 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 643.925606] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 643.925606] env[62736]: value = "task-397696" [ 643.925606] env[62736]: _type = "Task" [ 643.925606] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 643.933347] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397696, 'name': Rename_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 644.002232] env[62736]: INFO nova.compute.manager [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 558058e0-9808-415a-8866-75d1c8598d43] Took 1.03 seconds to deallocate network for instance. [ 644.004283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 24d65a77027a47f98634ed9b23c223ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.060897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24d65a77027a47f98634ed9b23c223ff [ 644.105220] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquiring lock "3135faf8-b918-42f5-bcaa-c23a19166bb3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.105442] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "3135faf8-b918-42f5-bcaa-c23a19166bb3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.177942] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 644.237340] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 644.238982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 3c86a1e0b0fd47b7a29d8a7ef24dc502 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.266730] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 644.267276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 1996ff51f3ab4985a6e1392ae8d4b96e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.278627] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1996ff51f3ab4985a6e1392ae8d4b96e [ 644.289811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c86a1e0b0fd47b7a29d8a7ef24dc502 [ 644.336134] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.577s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 644.336360] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 644.338031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 11de40ed7c4048888a8b7ee39c655ae4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.339038] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.535s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.340477] env[62736]: INFO nova.compute.claims [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 644.342017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 1cb94396decf4f42a31138401b0b2af8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.392563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cb94396decf4f42a31138401b0b2af8 [ 644.404970] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11de40ed7c4048888a8b7ee39c655ae4 [ 644.435666] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397696, 'name': Rename_Task, 'duration_secs': 0.152007} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 644.437013] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 644.439220] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-6fb79c39-46e6-4bea-bf16-61d9a0934578 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.441249] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "97927074-a676-4ebd-aa7a-fe92e3ce1be6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.441444] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "97927074-a676-4ebd-aa7a-fe92e3ce1be6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.446985] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 644.446985] env[62736]: value = "task-397697" [ 644.446985] env[62736]: _type = "Task" [ 644.446985] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 644.456232] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397697, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 644.509499] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg bc62afbe7846467cbc8a0496ede4ba1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.551409] env[62736]: DEBUG nova.compute.manager [req-69a99a83-905c-48d6-a8ec-374f9cfda7a2 req-fea27a6d-9891-4435-a9fa-8fef92ac74fc service nova] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Received event network-vif-deleted-08f341c7-922c-41cc-8984-e8f0675c276f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 644.553098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc62afbe7846467cbc8a0496ede4ba1d [ 644.757184] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.772991] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Releasing lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 644.773396] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 644.773649] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 644.773954] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bc3e265b-b8ca-485a-ae08-545fcb76c963 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.783326] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffbc27ac-0462-4968-a156-c921f94723a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 644.806570] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ce17af27-45c3-461e-8cb7-b9b023d88361 could not be found. [ 644.806785] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 644.806965] env[62736]: INFO nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Took 0.03 seconds to destroy the instance on the hypervisor. [ 644.807209] env[62736]: DEBUG oslo.service.loopingcall [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 644.807444] env[62736]: DEBUG nova.compute.manager [-] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 644.807556] env[62736]: DEBUG nova.network.neutron [-] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 644.822650] env[62736]: DEBUG nova.network.neutron [-] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 644.823185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 60affbc79e684af38038873f73864019 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.834212] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60affbc79e684af38038873f73864019 [ 644.844907] env[62736]: DEBUG nova.compute.utils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 644.845723] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 67b635b20e5d439da0c662766f07ad00 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.852286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg bccb2048b83b49ea80364c37bfbc7219 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 644.852286] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 644.852286] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 644.863120] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bccb2048b83b49ea80364c37bfbc7219 [ 644.865826] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67b635b20e5d439da0c662766f07ad00 [ 644.903699] env[62736]: DEBUG nova.policy [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '252ad3448fca4b76b3dce258c6f00107', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2848f00a047a4e638b81a83e20c4c182', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 644.917542] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "61f31838-2fd1-4804-82b7-56b3a08c3ff5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.917783] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "61f31838-2fd1-4804-82b7-56b3a08c3ff5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 644.959761] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397697, 'name': PowerOnVM_Task} progress is 88%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 645.039833] env[62736]: INFO nova.scheduler.client.report [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Deleted allocations for instance 558058e0-9808-415a-8866-75d1c8598d43 [ 645.056838] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg b6b3d9655d8f41faa14b9ceace4c07e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.068999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6b3d9655d8f41faa14b9ceace4c07e5 [ 645.253073] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Successfully created port: 012ccbce-db53-42bf-b39c-ede4b3b9b3fc {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 645.326183] env[62736]: DEBUG nova.network.neutron [-] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.326725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8fddfacdafef4f7e8a71bb39aafeeb4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.336919] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fddfacdafef4f7e8a71bb39aafeeb4d [ 645.350344] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 645.352176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 1a1c0df609d14106af3ae1affb4da1d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.397165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a1c0df609d14106af3ae1affb4da1d2 [ 645.461145] env[62736]: DEBUG oslo_vmware.api [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397697, 'name': PowerOnVM_Task, 'duration_secs': 0.722217} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 645.461398] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 645.461584] env[62736]: INFO nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Took 6.40 seconds to spawn the instance on the hypervisor. [ 645.461754] env[62736]: DEBUG nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 645.462624] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94658c8b-1666-43c7-9af3-c2077c2e77f4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.470533] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 21319039674d44548279ed3b1ab1c952 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.513533] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21319039674d44548279ed3b1ab1c952 [ 645.559124] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b1c7ae72-5848-4021-afd4-652131ec82bd tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "558058e0-9808-415a-8866-75d1c8598d43" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.602s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 645.559708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg eece1967db7f409a8994027998629cbd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.569920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eece1967db7f409a8994027998629cbd [ 645.829370] env[62736]: INFO nova.compute.manager [-] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Took 1.02 seconds to deallocate network for instance. [ 645.836816] env[62736]: DEBUG nova.compute.claims [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 645.837588] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.859708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 08caf5da2cb942a38422b878a30a0920 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 645.891702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08caf5da2cb942a38422b878a30a0920 [ 645.912899] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-183af9ce-b79f-48e4-a7f4-42f099d517b9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.922898] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3d48261-7a38-4433-856f-b7553d763e85 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.965072] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ef32e9f-12a5-4da3-ad63-8d70ed576ff1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.981328] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16d102b9-3e01-4ae5-945d-98f1c292bc88 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 645.986381] env[62736]: INFO nova.compute.manager [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Took 28.02 seconds to build instance. [ 645.986839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 232aaa345571472ea05a658b3e06577e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.002297] env[62736]: DEBUG nova.compute.provider_tree [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 646.002940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg c8b0139ff6d044e9ba8a4b3b9f71671d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.004093] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 232aaa345571472ea05a658b3e06577e [ 646.012644] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c8b0139ff6d044e9ba8a4b3b9f71671d [ 646.063442] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 646.065160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 95dc1e536bf242d0b85e349975c88dcb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.097545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95dc1e536bf242d0b85e349975c88dcb [ 646.157218] env[62736]: ERROR nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 646.157218] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 646.157218] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 646.157218] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 646.157218] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 646.157218] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 646.157218] env[62736]: ERROR nova.compute.manager raise self.value [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 646.157218] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 646.157218] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 646.157218] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 646.157709] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 646.157709] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 646.157709] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 646.157709] env[62736]: ERROR nova.compute.manager [ 646.157709] env[62736]: Traceback (most recent call last): [ 646.157709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 646.157709] env[62736]: listener.cb(fileno) [ 646.157709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 646.157709] env[62736]: result = function(*args, **kwargs) [ 646.157709] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 646.157709] env[62736]: return func(*args, **kwargs) [ 646.157709] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 646.157709] env[62736]: raise e [ 646.157709] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 646.157709] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 646.157709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 646.157709] env[62736]: created_port_ids = self._update_ports_for_instance( [ 646.157709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 646.157709] env[62736]: with excutils.save_and_reraise_exception(): [ 646.157709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 646.157709] env[62736]: self.force_reraise() [ 646.157709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 646.157709] env[62736]: raise self.value [ 646.157709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 646.157709] env[62736]: updated_port = self._update_port( [ 646.157709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 646.157709] env[62736]: _ensure_no_port_binding_failure(port) [ 646.157709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 646.157709] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 646.158487] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 646.158487] env[62736]: Removing descriptor: 16 [ 646.340874] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 379b8e7973fe47b08446eb6b93430408 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.353895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 379b8e7973fe47b08446eb6b93430408 [ 646.368083] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 646.390025] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 646.390416] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 646.390618] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 646.390878] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 646.391124] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 646.391323] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 646.391584] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 646.391825] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 646.392067] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 646.392309] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 646.392623] env[62736]: DEBUG nova.virt.hardware [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 646.394098] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38fb9855-7635-4b41-b581-029f975d7f97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.402254] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f15493d7-382d-477d-b284-f3200d455324 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.419176] env[62736]: ERROR nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Traceback (most recent call last): [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] yield resources [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.driver.spawn(context, instance, image_meta, [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self._vmops.spawn(context, instance, image_meta, injected_files, [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] vm_ref = self.build_virtual_machine(instance, [ 646.419176] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] vif_infos = vmwarevif.get_vif_info(self._session, [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] for vif in network_info: [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return self._sync_wrapper(fn, *args, **kwargs) [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.wait() [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self[:] = self._gt.wait() [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return self._exit_event.wait() [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 646.419677] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] current.throw(*self._exc) [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] result = function(*args, **kwargs) [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return func(*args, **kwargs) [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise e [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] nwinfo = self.network_api.allocate_for_instance( [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] created_port_ids = self._update_ports_for_instance( [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] with excutils.save_and_reraise_exception(): [ 646.420080] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.force_reraise() [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise self.value [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] updated_port = self._update_port( [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] _ensure_no_port_binding_failure(port) [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise exception.PortBindingFailed(port_id=port['id']) [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 646.420477] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] [ 646.420477] env[62736]: INFO nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Terminating instance [ 646.422531] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquiring lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 646.422758] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquired lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 646.422978] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 646.423475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 17bd7ebcf1ac4ceeaf910494e5d35d47 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.430461] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17bd7ebcf1ac4ceeaf910494e5d35d47 [ 646.434836] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.435277] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.492571] env[62736]: DEBUG oslo_concurrency.lockutils [None req-432886e9-c3a7-4fe4-bbf5-b0b28cdf5e39 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 76.657s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.493124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 83e3d8aa85e84f0cbc0b6b3b937f78e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.504660] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83e3d8aa85e84f0cbc0b6b3b937f78e0 [ 646.505591] env[62736]: DEBUG nova.scheduler.client.report [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 646.507971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 212ed0f9d4b94959893a940c0196f8a4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.518946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 212ed0f9d4b94959893a940c0196f8a4 [ 646.573944] env[62736]: DEBUG nova.compute.manager [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Received event network-changed-012ccbce-db53-42bf-b39c-ede4b3b9b3fc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 646.574191] env[62736]: DEBUG nova.compute.manager [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Refreshing instance network info cache due to event network-changed-012ccbce-db53-42bf-b39c-ede4b3b9b3fc. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 646.574331] env[62736]: DEBUG oslo_concurrency.lockutils [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] Acquiring lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 646.581875] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.842973] env[62736]: INFO nova.compute.manager [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Rebuilding instance [ 646.883206] env[62736]: DEBUG nova.compute.manager [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 646.884199] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60351ead-7058-4885-abc3-41d5d201e6c5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 646.892476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 0aca6994bde54a1fb568475bf592f09e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 646.934091] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0aca6994bde54a1fb568475bf592f09e [ 646.941745] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 646.995079] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 646.996951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 9ccb72d9c7aa4989b7075947c9266929 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.012046] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.673s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.012903] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 647.014717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg f5404102ddbc4f1ba7185abf81ec020e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.015879] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.023s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.017577] env[62736]: INFO nova.compute.claims [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 647.019552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 40fafb48a693446784c66cf8784abec5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.039874] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 647.040570] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg ab1f9eb8c4b44184aa117a1ecc5c23db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.049126] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ccb72d9c7aa4989b7075947c9266929 [ 647.054978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5404102ddbc4f1ba7185abf81ec020e [ 647.056759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab1f9eb8c4b44184aa117a1ecc5c23db [ 647.059504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40fafb48a693446784c66cf8784abec5 [ 647.396339] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 647.396780] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-31e28488-ff9c-4e79-8768-39e656d9176e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.403877] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 647.403877] env[62736]: value = "task-397698" [ 647.403877] env[62736]: _type = "Task" [ 647.403877] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 647.411905] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397698, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 647.516258] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.522667] env[62736]: DEBUG nova.compute.utils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 647.523316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 8890e920b6fc4709a91cc5d5dcf115a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.525386] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg ada6f0b9f09a42a0b7ce4a09846fb8c2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.526923] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 647.527140] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 647.534726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ada6f0b9f09a42a0b7ce4a09846fb8c2 [ 647.538010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8890e920b6fc4709a91cc5d5dcf115a1 [ 647.542268] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Releasing lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 647.542645] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 647.543050] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 647.543129] env[62736]: DEBUG oslo_concurrency.lockutils [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] Acquired lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 647.543251] env[62736]: DEBUG nova.network.neutron [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Refreshing network info cache for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 647.543686] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] Expecting reply to msg 50926cb71d52417e8c53f41fb43b7db7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.544833] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5295945e-c1f0-48c7-821c-ab1cb870a139 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.549946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50926cb71d52417e8c53f41fb43b7db7 [ 647.554646] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf68e1dc-f5bd-480f-8efc-2f9e217f9b55 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.576183] env[62736]: DEBUG nova.policy [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6a6d2a917fec48e48e943137dff043fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd781d0a045444391bc070520be1b6431', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 647.580891] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9a445d5c-7a5f-4149-b198-306a04b4c040 could not be found. [ 647.581101] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 647.581279] env[62736]: INFO nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Took 0.04 seconds to destroy the instance on the hypervisor. [ 647.581513] env[62736]: DEBUG oslo.service.loopingcall [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 647.581813] env[62736]: DEBUG nova.compute.manager [-] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 647.581907] env[62736]: DEBUG nova.network.neutron [-] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 647.603468] env[62736]: DEBUG nova.network.neutron [-] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 647.603912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6d8d5dcbfee74388a6f626bc36bfd841 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 647.610054] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d8d5dcbfee74388a6f626bc36bfd841 [ 647.880728] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Successfully created port: 8a271f17-c894-4132-ac3d-e76ca3e8fbfa {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 647.913721] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397698, 'name': PowerOffVM_Task, 'duration_secs': 0.170953} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 647.913970] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 647.914172] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 647.914927] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fec6dec-58e3-48fe-b769-1f9a6f763bfe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.921317] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 647.921529] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-9387be8f-fcb1-483f-bb03-3e7627d688f8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.945958] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 647.946197] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 647.946369] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleting the datastore file [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 647.946616] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ec094052-243f-4ad2-a86b-899711c9b37c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 647.953958] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 647.953958] env[62736]: value = "task-397700" [ 647.953958] env[62736]: _type = "Task" [ 647.953958] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 647.962918] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397700, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 648.027871] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 648.029818] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg fcc705bf6cd046e6b49512f6256e5785 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 648.080425] env[62736]: DEBUG nova.network.neutron [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 648.098817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fcc705bf6cd046e6b49512f6256e5785 [ 648.105449] env[62736]: DEBUG nova.network.neutron [-] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 648.105980] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 473728a7ebd34392be570c720192b392 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 648.114664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 473728a7ebd34392be570c720192b392 [ 648.222187] env[62736]: DEBUG nova.network.neutron [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 648.222761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] Expecting reply to msg 94290a98499d4b62bd54f5d2912ccbe7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 648.235152] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94290a98499d4b62bd54f5d2912ccbe7 [ 648.476166] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397700, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.136864} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 648.479046] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 648.479220] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 648.479432] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 648.481698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8006c3967d4d4347bce66b95ac35d01f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 648.522021] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8006c3967d4d4347bce66b95ac35d01f [ 648.534061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg ee0e90c7a3c441f4b42d579e6d813a7f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 648.563558] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b87ff28-65b5-45ed-9667-dc8003a5a944 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.567262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee0e90c7a3c441f4b42d579e6d813a7f [ 648.571218] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab6a7881-88b3-4379-b75a-0fd58f462fa0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.292876] env[62736]: INFO nova.compute.manager [-] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Took 1.71 seconds to deallocate network for instance. [ 649.294276] env[62736]: DEBUG oslo_concurrency.lockutils [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] Releasing lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 649.294695] env[62736]: DEBUG nova.compute.manager [req-e9f0d267-00ae-4387-87ca-8028e6bbc137 req-a91cbad3-707e-425f-bfd3-71d8ca431afc service nova] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Received event network-vif-deleted-012ccbce-db53-42bf-b39c-ede4b3b9b3fc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 649.296816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg d0bc988d2dd442f59efbdab4b9ea3f59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 649.298292] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 649.310990] env[62736]: DEBUG nova.compute.manager [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Received event network-changed-8a271f17-c894-4132-ac3d-e76ca3e8fbfa {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 649.311175] env[62736]: DEBUG nova.compute.manager [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Refreshing instance network info cache due to event network-changed-8a271f17-c894-4132-ac3d-e76ca3e8fbfa. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 649.311376] env[62736]: DEBUG oslo_concurrency.lockutils [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] Acquiring lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 649.311514] env[62736]: DEBUG oslo_concurrency.lockutils [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] Acquired lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 649.311670] env[62736]: DEBUG nova.network.neutron [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Refreshing network info cache for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 649.312243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] Expecting reply to msg da6a4dbd530a42a19800b9c5e87e7157 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 649.318347] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a0beea3-dab7-4df7-91ab-6708d971b36a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.319151] env[62736]: DEBUG nova.compute.claims [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 649.319347] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 649.320908] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da6a4dbd530a42a19800b9c5e87e7157 [ 649.328491] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a30eb2fc-d787-4a70-a523-f6c7becb8f23 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.332800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0bc988d2dd442f59efbdab4b9ea3f59 [ 649.338726] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 649.338948] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 649.339099] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 649.339284] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 649.339444] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 649.339593] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 649.339790] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 649.339938] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 649.340110] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 649.341163] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 649.341163] env[62736]: DEBUG nova.virt.hardware [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 649.341642] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-008d7387-edb6-41a5-ba0f-40bc48df099e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.352613] env[62736]: DEBUG nova.compute.provider_tree [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 649.353160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg e3bce9d5051140abbaeefc83d9bc65dc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 649.359238] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42a2c892-c63c-49b5-9f50-a0404e3c123b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.363188] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3bce9d5051140abbaeefc83d9bc65dc [ 649.529246] env[62736]: ERROR nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 649.529246] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 649.529246] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 649.529246] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 649.529246] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 649.529246] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 649.529246] env[62736]: ERROR nova.compute.manager raise self.value [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 649.529246] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 649.529246] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 649.529246] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 649.529766] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 649.529766] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 649.529766] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 649.529766] env[62736]: ERROR nova.compute.manager [ 649.529766] env[62736]: Traceback (most recent call last): [ 649.529766] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 649.529766] env[62736]: listener.cb(fileno) [ 649.529766] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 649.529766] env[62736]: result = function(*args, **kwargs) [ 649.529766] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 649.529766] env[62736]: return func(*args, **kwargs) [ 649.529766] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 649.529766] env[62736]: raise e [ 649.529766] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 649.529766] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 649.529766] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 649.529766] env[62736]: created_port_ids = self._update_ports_for_instance( [ 649.529766] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 649.529766] env[62736]: with excutils.save_and_reraise_exception(): [ 649.529766] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 649.529766] env[62736]: self.force_reraise() [ 649.529766] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 649.529766] env[62736]: raise self.value [ 649.529766] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 649.529766] env[62736]: updated_port = self._update_port( [ 649.529766] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 649.529766] env[62736]: _ensure_no_port_binding_failure(port) [ 649.529766] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 649.529766] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 649.530540] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 649.530540] env[62736]: Removing descriptor: 16 [ 649.530540] env[62736]: ERROR nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Traceback (most recent call last): [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] yield resources [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.driver.spawn(context, instance, image_meta, [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self._vmops.spawn(context, instance, image_meta, injected_files, [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 649.530540] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] vm_ref = self.build_virtual_machine(instance, [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] vif_infos = vmwarevif.get_vif_info(self._session, [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] for vif in network_info: [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self._sync_wrapper(fn, *args, **kwargs) [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.wait() [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self[:] = self._gt.wait() [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self._exit_event.wait() [ 649.530872] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] result = hub.switch() [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self.greenlet.switch() [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] result = function(*args, **kwargs) [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return func(*args, **kwargs) [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise e [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] nwinfo = self.network_api.allocate_for_instance( [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 649.531237] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] created_port_ids = self._update_ports_for_instance( [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] with excutils.save_and_reraise_exception(): [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.force_reraise() [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise self.value [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] updated_port = self._update_port( [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] _ensure_no_port_binding_failure(port) [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 649.531605] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise exception.PortBindingFailed(port_id=port['id']) [ 649.531952] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 649.531952] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] [ 649.531952] env[62736]: INFO nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Terminating instance [ 649.531952] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 649.833269] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 649.833536] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 649.833693] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 649.833869] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 649.834004] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 649.834140] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 649.834343] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 649.834489] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 649.834754] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 649.834894] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 649.834999] env[62736]: DEBUG nova.virt.hardware [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 649.835881] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2bc0aae-8c60-43fd-a811-064d6184c5cf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.838947] env[62736]: DEBUG nova.network.neutron [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.846018] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f068d79a-95cd-4205-a33a-d3ad8071c287 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.859893] env[62736]: DEBUG nova.scheduler.client.report [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 649.862467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 0ca7d98d38944ea789da05ae5605cc94 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 649.863496] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 649.869175] env[62736]: DEBUG oslo.service.loopingcall [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 649.869681] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 649.869951] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f21d1fae-3c7e-4f59-a630-d7b41026f97a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.884416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ca7d98d38944ea789da05ae5605cc94 [ 649.890235] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 649.890235] env[62736]: value = "task-397701" [ 649.890235] env[62736]: _type = "Task" [ 649.890235] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 649.897755] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397701, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 649.921052] env[62736]: DEBUG nova.network.neutron [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 649.921855] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] Expecting reply to msg e089aac6e25a4b13827166b0309307c0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 649.930658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e089aac6e25a4b13827166b0309307c0 [ 650.373950] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 3.355s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.373950] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 650.373950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 5d5f0f68308641a388fbfad50e7ebd46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 650.388115] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 18.640s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.388115] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.388115] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 650.388115] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.512s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.388115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 8525eb5418ed430088ba2bdc2459fa44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 650.389663] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccb75009-3ab8-4a2d-a444-28947546743f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.405012] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-379fe335-2e5d-4460-be88-cde226ed7d60 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.408988] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397701, 'name': CreateVM_Task, 'duration_secs': 0.237546} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 650.409348] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 650.410165] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.410471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.410859] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 650.411171] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b966a748-ec1b-4d34-8f33-78db4c880bfa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.421784] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d5f0f68308641a388fbfad50e7ebd46 [ 650.423087] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3dcdf6fe-88b7-43fc-b0d0-2633e609ad51 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.425943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8525eb5418ed430088ba2bdc2459fa44 [ 650.427554] env[62736]: DEBUG oslo_concurrency.lockutils [req-4575dc84-60ad-464d-a978-d52a0b3dde21 req-fdc947ae-98e8-4ab1-be32-4e576cbb137b service nova] Releasing lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 650.428149] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 650.428149] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]528bb3c6-2853-f00d-7fec-5fc601158427" [ 650.428149] env[62736]: _type = "Task" [ 650.428149] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 650.428817] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.429116] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 650.429757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg a7afb69c9f464f2c9c46c4ec3b223164 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 650.437100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7afb69c9f464f2c9c46c4ec3b223164 [ 650.438551] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2655fc19-bcc0-4948-8028-3ff4b9c8dc65 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.445111] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]528bb3c6-2853-f00d-7fec-5fc601158427, 'name': SearchDatastore_Task, 'duration_secs': 0.008658} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 650.445780] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 650.446131] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 650.446473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 650.446724] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 650.446997] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 650.471900] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bf1d5d3f-bf47-4d63-a8e4-013c53f96aae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.474123] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181750MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 650.474400] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.480360] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 650.480701] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 650.481463] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-debc45e5-dad7-4e6e-acfb-726557f450e0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.487387] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 650.487387] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52199381-aed9-d7d4-c87d-c88422a8dc92" [ 650.487387] env[62736]: _type = "Task" [ 650.487387] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 650.495281] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52199381-aed9-d7d4-c87d-c88422a8dc92, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 650.815396] env[62736]: DEBUG nova.compute.manager [req-af865d9c-d1f0-47d5-a91b-9c1b98214056 req-4e2b353d-1860-47c1-96a6-96703178f0a3 service nova] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Received event network-vif-deleted-8a271f17-c894-4132-ac3d-e76ca3e8fbfa {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 650.886477] env[62736]: DEBUG nova.compute.utils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 650.887380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg f4c9e19ff94349e6a1e56d3e68b75baa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 650.888659] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 650.889059] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 650.904264] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4c9e19ff94349e6a1e56d3e68b75baa [ 650.934606] env[62736]: DEBUG nova.policy [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cf477a6798c9449bbb615277de8c1170', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2b6ce1f734f74a948ee720d8b43795b0', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 650.958361] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.006829] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52199381-aed9-d7d4-c87d-c88422a8dc92, 'name': SearchDatastore_Task, 'duration_secs': 0.007782} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 651.007667] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-69d75911-ded5-4ab3-887b-eafdbcf9dd9a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.016534] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 651.016534] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]528e3217-8b84-3be0-397e-efd33a09c08d" [ 651.016534] env[62736]: _type = "Task" [ 651.016534] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 651.030114] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]528e3217-8b84-3be0-397e-efd33a09c08d, 'name': SearchDatastore_Task, 'duration_secs': 0.00965} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 651.030341] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 651.030589] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 651.030830] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-eabbc5c5-30bd-4a6d-96e0-b80bdd8335e4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.037657] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 651.037657] env[62736]: value = "task-397702" [ 651.037657] env[62736]: _type = "Task" [ 651.037657] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 651.048165] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397702, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 651.072645] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.073095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg d4ded7febc1c4c0d8b6b1c57c0513de7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 651.083475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4ded7febc1c4c0d8b6b1c57c0513de7 [ 651.249976] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Successfully created port: 04b56e77-5706-4977-9ead-92b48295fcb5 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 651.401037] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 651.402759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 58fd8ea2e10d42aa96bc4e00275a3fc0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 651.446232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58fd8ea2e10d42aa96bc4e00275a3fc0 [ 651.493662] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ded7c219-c8a6-44e7-81ce-ffbd85953bb6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.501670] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef5d70b0-9e61-4064-aff5-941def76742d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.534950] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2793c60-e22e-4467-8c3c-d74eb295384d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.550901] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81992f70-4080-49b5-8cf1-92e9924059c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.559623] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397702, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.47559} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 651.567451] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 651.567676] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 651.568146] env[62736]: DEBUG nova.compute.provider_tree [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 651.568613] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 238dc0df5a264f36bddf2e73a6e85f60 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 651.569427] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-867b2d2c-0657-404f-9a5b-e36bf36befc5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.575888] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 651.576292] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 651.576537] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 651.576863] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 651.576863] env[62736]: value = "task-397703" [ 651.576863] env[62736]: _type = "Task" [ 651.576863] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 651.577407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 238dc0df5a264f36bddf2e73a6e85f60 [ 651.577760] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0a6e9c41-211e-4788-a093-fa929bf37a34 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.587828] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397703, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 651.591729] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25ecb2ba-8115-447f-8fd3-e823cc49cb0e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 651.615071] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ad008294-52db-40de-95ad-3e0f2d30a462 could not be found. [ 651.615443] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 651.615634] env[62736]: INFO nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Took 0.04 seconds to destroy the instance on the hypervisor. [ 651.615879] env[62736]: DEBUG oslo.service.loopingcall [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 651.616120] env[62736]: DEBUG nova.compute.manager [-] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 651.616208] env[62736]: DEBUG nova.network.neutron [-] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 651.630381] env[62736]: DEBUG nova.network.neutron [-] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 651.630996] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3fcca8d82d0247f39820e013e8a77c4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 651.638510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fcca8d82d0247f39820e013e8a77c4c [ 651.906941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg f125a4c14e884d9da2eb541cb3b4e457 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 651.948920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f125a4c14e884d9da2eb541cb3b4e457 [ 652.072928] env[62736]: DEBUG nova.scheduler.client.report [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 652.075377] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 46d3956e9f794cac87f0ad2173eb07ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 652.089631] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397703, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.063125} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 652.090184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46d3956e9f794cac87f0ad2173eb07ea [ 652.090604] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 652.091365] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37a17142-a253-4669-87f9-35161fe714f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.111041] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Reconfiguring VM instance instance-0000001a to attach disk [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 652.111511] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-4cd4b4e9-97a3-4cea-9f93-c2b758200955 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.132195] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 652.132195] env[62736]: value = "task-397704" [ 652.132195] env[62736]: _type = "Task" [ 652.132195] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 652.132523] env[62736]: DEBUG nova.network.neutron [-] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 652.132926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ba05dabc46ea4281852d490a57cada84 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 652.141519] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397704, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 652.141999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba05dabc46ea4281852d490a57cada84 [ 652.142982] env[62736]: ERROR nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 652.142982] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.142982] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.142982] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.142982] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.142982] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.142982] env[62736]: ERROR nova.compute.manager raise self.value [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.142982] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 652.142982] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.142982] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 652.143625] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.143625] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 652.143625] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 652.143625] env[62736]: ERROR nova.compute.manager [ 652.143625] env[62736]: Traceback (most recent call last): [ 652.143625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 652.143625] env[62736]: listener.cb(fileno) [ 652.143625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 652.143625] env[62736]: result = function(*args, **kwargs) [ 652.143625] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 652.143625] env[62736]: return func(*args, **kwargs) [ 652.143625] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 652.143625] env[62736]: raise e [ 652.143625] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.143625] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 652.143625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.143625] env[62736]: created_port_ids = self._update_ports_for_instance( [ 652.143625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.143625] env[62736]: with excutils.save_and_reraise_exception(): [ 652.143625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.143625] env[62736]: self.force_reraise() [ 652.143625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.143625] env[62736]: raise self.value [ 652.143625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.143625] env[62736]: updated_port = self._update_port( [ 652.143625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.143625] env[62736]: _ensure_no_port_binding_failure(port) [ 652.143625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.143625] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 652.144788] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 652.144788] env[62736]: Removing descriptor: 16 [ 652.410226] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 652.433911] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 652.434161] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 652.434317] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 652.434496] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 652.434635] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 652.434787] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 652.434969] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 652.435104] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 652.435262] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 652.435415] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 652.435586] env[62736]: DEBUG nova.virt.hardware [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 652.436459] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8d1b59e-9825-4b7a-8faa-d90822425019 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.444017] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2ded6b5-68f6-4b71-b514-98aa5d93b5f7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.458687] env[62736]: ERROR nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] Traceback (most recent call last): [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] yield resources [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.driver.spawn(context, instance, image_meta, [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self._vmops.spawn(context, instance, image_meta, injected_files, [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] vm_ref = self.build_virtual_machine(instance, [ 652.458687] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] vif_infos = vmwarevif.get_vif_info(self._session, [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] for vif in network_info: [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return self._sync_wrapper(fn, *args, **kwargs) [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.wait() [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self[:] = self._gt.wait() [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return self._exit_event.wait() [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 652.459030] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] current.throw(*self._exc) [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] result = function(*args, **kwargs) [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return func(*args, **kwargs) [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise e [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] nwinfo = self.network_api.allocate_for_instance( [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] created_port_ids = self._update_ports_for_instance( [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] with excutils.save_and_reraise_exception(): [ 652.459380] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.force_reraise() [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise self.value [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] updated_port = self._update_port( [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] _ensure_no_port_binding_failure(port) [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise exception.PortBindingFailed(port_id=port['id']) [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 652.459739] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] [ 652.459739] env[62736]: INFO nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Terminating instance [ 652.461035] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquiring lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 652.461193] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquired lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 652.461354] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 652.461764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 7d38e45eb9204c00bb04668bf6d0a0c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 652.468604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d38e45eb9204c00bb04668bf6d0a0c6 [ 652.580040] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.192s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.580040] env[62736]: ERROR nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Traceback (most recent call last): [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.driver.spawn(context, instance, image_meta, [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 652.580040] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] vm_ref = self.build_virtual_machine(instance, [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] vif_infos = vmwarevif.get_vif_info(self._session, [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] for vif in network_info: [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self._sync_wrapper(fn, *args, **kwargs) [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.wait() [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self[:] = self._gt.wait() [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self._exit_event.wait() [ 652.580440] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] result = hub.switch() [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return self.greenlet.switch() [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] result = function(*args, **kwargs) [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] return func(*args, **kwargs) [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise e [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] nwinfo = self.network_api.allocate_for_instance( [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 652.580809] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] created_port_ids = self._update_ports_for_instance( [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] with excutils.save_and_reraise_exception(): [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] self.force_reraise() [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise self.value [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] updated_port = self._update_port( [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] _ensure_no_port_binding_failure(port) [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 652.581169] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] raise exception.PortBindingFailed(port_id=port['id']) [ 652.581516] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] nova.exception.PortBindingFailed: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. [ 652.581516] env[62736]: ERROR nova.compute.manager [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] [ 652.581516] env[62736]: DEBUG nova.compute.utils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 652.581516] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.901s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.582522] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg ca932a2489904dcd9b5dea103528abc1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 652.583766] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Build of instance a3182b4a-1846-4766-8988-aab0a74fa4c8 was re-scheduled: Binding failed for port 0b41917f-afed-4e53-b215-68a6e91bbd7d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 652.584205] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 652.584426] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquiring lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 652.584566] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Acquired lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 652.584715] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 652.585074] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg cb51331b450949388f3eb7239156f7eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 652.591621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb51331b450949388f3eb7239156f7eb [ 652.613902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca932a2489904dcd9b5dea103528abc1 [ 652.637989] env[62736]: INFO nova.compute.manager [-] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Took 1.02 seconds to deallocate network for instance. [ 652.642862] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397704, 'name': ReconfigVM_Task, 'duration_secs': 0.256023} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 652.643343] env[62736]: DEBUG nova.compute.claims [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 652.643493] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.643748] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Reconfigured VM instance instance-0000001a to attach disk [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7/13247e87-08ab-434e-b091-8b6116280fb7.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 652.644322] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-f1b83426-4072-45c6-9ec9-257361d72b48 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.650675] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 652.650675] env[62736]: value = "task-397705" [ 652.650675] env[62736]: _type = "Task" [ 652.650675] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 652.659206] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397705, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 652.839083] env[62736]: DEBUG nova.compute.manager [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Received event network-changed-04b56e77-5706-4977-9ead-92b48295fcb5 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 652.839247] env[62736]: DEBUG nova.compute.manager [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Refreshing instance network info cache due to event network-changed-04b56e77-5706-4977-9ead-92b48295fcb5. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 652.839506] env[62736]: DEBUG oslo_concurrency.lockutils [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] Acquiring lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 652.979071] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.055199] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 653.055728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 21e5d744a35241f4ba7d45353575e213 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.063892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21e5d744a35241f4ba7d45353575e213 [ 653.107710] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.155557] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 653.156082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 828d1109676c44a4a9dc8d2658c26e09 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.163680] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397705, 'name': Rename_Task, 'duration_secs': 0.124965} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 653.163940] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 653.164463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 828d1109676c44a4a9dc8d2658c26e09 [ 653.164806] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-8701d4f3-b939-42c8-a7e6-38c8f8de46c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.171280] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 653.171280] env[62736]: value = "task-397706" [ 653.171280] env[62736]: _type = "Task" [ 653.171280] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 653.179189] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397706, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 653.470076] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a89e3652-81c6-4993-9a04-ee11b75704d0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.478230] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dbf5a55-532b-48bc-99e4-471b41326391 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.510282] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae6ac060-753b-4522-84fb-abdbcd456d9c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.517487] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-895f4f8a-aea2-47b2-a748-52c3561fd516 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.531062] env[62736]: DEBUG nova.compute.provider_tree [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 653.531559] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 1bf090b3f4e44f2aacdfb2d9a52c84f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.539257] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bf090b3f4e44f2aacdfb2d9a52c84f9 [ 653.557738] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Releasing lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 653.558144] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 653.558334] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 653.558626] env[62736]: DEBUG oslo_concurrency.lockutils [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] Acquired lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 653.558793] env[62736]: DEBUG nova.network.neutron [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Refreshing network info cache for port 04b56e77-5706-4977-9ead-92b48295fcb5 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 653.559219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] Expecting reply to msg c635bef4e7cc480f961767d3a8268a89 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.560342] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8c73b9bc-b03b-43a9-854a-20b3e7a4e7d3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.566326] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c635bef4e7cc480f961767d3a8268a89 [ 653.569281] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e43310d-29f2-4d3c-9e64-dd1558a898e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.591290] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 38767b46-9a31-4031-981b-424497dc4a34 could not be found. [ 653.591503] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 653.591732] env[62736]: INFO nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Took 0.03 seconds to destroy the instance on the hypervisor. [ 653.591970] env[62736]: DEBUG oslo.service.loopingcall [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 653.592196] env[62736]: DEBUG nova.compute.manager [-] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 653.592286] env[62736]: DEBUG nova.network.neutron [-] [instance: 38767b46-9a31-4031-981b-424497dc4a34] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 653.606032] env[62736]: DEBUG nova.network.neutron [-] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.606503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6dc7929b6a4a4005860cda4bd6ecd4e4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.613557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6dc7929b6a4a4005860cda4bd6ecd4e4 [ 653.659048] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Releasing lock "refresh_cache-a3182b4a-1846-4766-8988-aab0a74fa4c8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 653.659414] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 653.659605] env[62736]: DEBUG nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 653.659772] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 653.677673] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 653.678127] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg a90ed9532c484854aae815e942d21a80 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.682024] env[62736]: DEBUG oslo_vmware.api [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397706, 'name': PowerOnVM_Task, 'duration_secs': 0.397816} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 653.682460] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 653.682659] env[62736]: DEBUG nova.compute.manager [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 653.683403] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9528fc05-8962-4d90-a755-9656fa1c1a37 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.686588] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a90ed9532c484854aae815e942d21a80 [ 653.691135] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg aa99aab7431f49668bf16db30638034f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 653.720589] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa99aab7431f49668bf16db30638034f [ 654.035077] env[62736]: DEBUG nova.scheduler.client.report [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 654.038095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 581764c790564c199e7dd40e81284769 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.052036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 581764c790564c199e7dd40e81284769 [ 654.078186] env[62736]: DEBUG nova.network.neutron [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 654.108847] env[62736]: DEBUG nova.network.neutron [-] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.109366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 18a0f8a40207474c9e7519a517c18e90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.118189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18a0f8a40207474c9e7519a517c18e90 [ 654.137666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg beddc383e3174a719b632e031d9f98d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.147546] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg beddc383e3174a719b632e031d9f98d6 [ 654.148697] env[62736]: DEBUG nova.network.neutron [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.149051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] Expecting reply to msg 48e9172194104a9d855ef02b93a09fd5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.156523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48e9172194104a9d855ef02b93a09fd5 [ 654.183499] env[62736]: DEBUG nova.network.neutron [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 654.184476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 5207f1f8f8744f3895a0d2dd303eb0ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.192486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5207f1f8f8744f3895a0d2dd303eb0ab [ 654.201143] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.540562] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.960s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 654.541203] env[62736]: ERROR nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Traceback (most recent call last): [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.driver.spawn(context, instance, image_meta, [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self._vmops.spawn(context, instance, image_meta, injected_files, [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] vm_ref = self.build_virtual_machine(instance, [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] vif_infos = vmwarevif.get_vif_info(self._session, [ 654.541203] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] for vif in network_info: [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self._sync_wrapper(fn, *args, **kwargs) [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.wait() [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self[:] = self._gt.wait() [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self._exit_event.wait() [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] result = hub.switch() [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 654.541505] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return self.greenlet.switch() [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] result = function(*args, **kwargs) [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] return func(*args, **kwargs) [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise e [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] nwinfo = self.network_api.allocate_for_instance( [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] created_port_ids = self._update_ports_for_instance( [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] with excutils.save_and_reraise_exception(): [ 654.541829] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] self.force_reraise() [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise self.value [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] updated_port = self._update_port( [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] _ensure_no_port_binding_failure(port) [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] raise exception.PortBindingFailed(port_id=port['id']) [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] nova.exception.PortBindingFailed: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. [ 654.542150] env[62736]: ERROR nova.compute.manager [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] [ 654.542412] env[62736]: DEBUG nova.compute.utils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 654.543289] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.850s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.545193] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 40be412a963d4b509a3bb937910b3d94 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.546427] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Build of instance c97b25ac-527f-404e-a539-1c0f43bcd66b was re-scheduled: Binding failed for port f755d6f1-e7ae-4c6f-9417-ff0e25adb144, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 654.546849] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 654.547064] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquiring lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.547199] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Acquired lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.547345] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 654.547697] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 5372f8398cd845f5b16858284cd1ccf8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.553707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5372f8398cd845f5b16858284cd1ccf8 [ 654.576181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40be412a963d4b509a3bb937910b3d94 [ 654.611947] env[62736]: INFO nova.compute.manager [-] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Took 1.02 seconds to deallocate network for instance. [ 654.614191] env[62736]: DEBUG nova.compute.claims [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 654.614364] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.640914] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "13247e87-08ab-434e-b091-8b6116280fb7" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.641142] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.641338] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "13247e87-08ab-434e-b091-8b6116280fb7-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.641515] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.641682] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 654.643645] env[62736]: INFO nova.compute.manager [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Terminating instance [ 654.645170] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "refresh_cache-13247e87-08ab-434e-b091-8b6116280fb7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.645346] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "refresh_cache-13247e87-08ab-434e-b091-8b6116280fb7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.645481] env[62736]: DEBUG nova.network.neutron [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 654.645861] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg ee5cc760ec99454291b7c3729dfb06e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.651017] env[62736]: DEBUG oslo_concurrency.lockutils [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] Releasing lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 654.651221] env[62736]: DEBUG nova.compute.manager [req-16f95c7c-2252-4f6c-abd7-e9a07981686c req-448f6b16-4869-4e3c-a63a-0355b2f3d600 service nova] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Received event network-vif-deleted-04b56e77-5706-4977-9ead-92b48295fcb5 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 654.652164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee5cc760ec99454291b7c3729dfb06e9 [ 654.686728] env[62736]: INFO nova.compute.manager [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] [instance: a3182b4a-1846-4766-8988-aab0a74fa4c8] Took 1.03 seconds to deallocate network for instance. [ 654.688445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 66cfa864eb8f4d10bacb85e1b6bcf728 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 654.740954] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66cfa864eb8f4d10bacb85e1b6bcf728 [ 655.069244] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.135935] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.135935] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 5a2a33cd76ab450a82bc3badce8241d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.146033] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a2a33cd76ab450a82bc3badce8241d6 [ 655.166317] env[62736]: DEBUG nova.network.neutron [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.192691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg 3da2ba1771d147b696eb29a04beabdff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.217205] env[62736]: DEBUG nova.network.neutron [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.217781] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg ec3e6de044c24aacb0443d3967218296 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.227128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec3e6de044c24aacb0443d3967218296 [ 655.233307] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3da2ba1771d147b696eb29a04beabdff [ 655.445452] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28b4d627-a8da-438f-8df9-c2a8fb67446e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.453673] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a8b6630-ec6e-4461-adf9-a5248699aab6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.484207] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-127c4ad1-a315-42f8-9489-abc7aafb20c8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.491372] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0536d399-238f-42e5-ac62-8abad089ca0f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.504362] env[62736]: DEBUG nova.compute.provider_tree [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 655.504870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 27af307f428e492cae1f5ca26d077392 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.511960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27af307f428e492cae1f5ca26d077392 [ 655.637190] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Releasing lock "refresh_cache-c97b25ac-527f-404e-a539-1c0f43bcd66b" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.637582] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 655.637582] env[62736]: DEBUG nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 655.637735] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 655.652075] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.652632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 02d03d0890df43de94572b815092df38 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.659711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02d03d0890df43de94572b815092df38 [ 655.714047] env[62736]: INFO nova.scheduler.client.report [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Deleted allocations for instance a3182b4a-1846-4766-8988-aab0a74fa4c8 [ 655.719752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Expecting reply to msg af151f4e21b24aba948717bacab389bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 655.720715] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "refresh_cache-13247e87-08ab-434e-b091-8b6116280fb7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.721133] env[62736]: DEBUG nova.compute.manager [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 655.721337] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 655.722314] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07643ab2-abeb-4b52-9c19-7d41d7931c42 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.730708] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 655.730708] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-810d770c-72e7-440f-9bf4-25989e6aea12 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.737509] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 655.737509] env[62736]: value = "task-397707" [ 655.737509] env[62736]: _type = "Task" [ 655.737509] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 655.741833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af151f4e21b24aba948717bacab389bd [ 655.745691] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397707, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 656.007271] env[62736]: DEBUG nova.scheduler.client.report [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 656.009792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 20ee906f751941b6a24dd83cc19981ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.020889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20ee906f751941b6a24dd83cc19981ae [ 656.156425] env[62736]: DEBUG nova.network.neutron [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.157207] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 15ac3e81de5242c2bdd396b470cf9906 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.173004] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15ac3e81de5242c2bdd396b470cf9906 [ 656.228650] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2f447a51-dc80-4fa4-a769-be1c0cdb279c tempest-ServerDiagnosticsTest-2135904566 tempest-ServerDiagnosticsTest-2135904566-project-member] Lock "a3182b4a-1846-4766-8988-aab0a74fa4c8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 91.780s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.229282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 0df9907856e64d669fe2c70bf1b1abe7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.244440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0df9907856e64d669fe2c70bf1b1abe7 [ 656.248310] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397707, 'name': PowerOffVM_Task, 'duration_secs': 0.175983} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 656.248652] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 656.248879] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 656.249200] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-6498bd0a-f5c8-403b-865b-3a2942880459 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.274449] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 656.274797] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 656.275040] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleting the datastore file [datastore1] 13247e87-08ab-434e-b091-8b6116280fb7 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 656.275350] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bd850b69-57aa-4349-b6f0-e8293b56dafa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.282241] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 656.282241] env[62736]: value = "task-397709" [ 656.282241] env[62736]: _type = "Task" [ 656.282241] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 656.291944] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397709, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 656.513300] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.970s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.514035] env[62736]: ERROR nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Traceback (most recent call last): [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.driver.spawn(context, instance, image_meta, [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self._vmops.spawn(context, instance, image_meta, injected_files, [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] vm_ref = self.build_virtual_machine(instance, [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] vif_infos = vmwarevif.get_vif_info(self._session, [ 656.514035] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] for vif in network_info: [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return self._sync_wrapper(fn, *args, **kwargs) [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.wait() [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self[:] = self._gt.wait() [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return self._exit_event.wait() [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] current.throw(*self._exc) [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 656.514370] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] result = function(*args, **kwargs) [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] return func(*args, **kwargs) [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise e [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] nwinfo = self.network_api.allocate_for_instance( [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] created_port_ids = self._update_ports_for_instance( [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] with excutils.save_and_reraise_exception(): [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] self.force_reraise() [ 656.514887] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise self.value [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] updated_port = self._update_port( [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] _ensure_no_port_binding_failure(port) [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] raise exception.PortBindingFailed(port_id=port['id']) [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] nova.exception.PortBindingFailed: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. [ 656.515369] env[62736]: ERROR nova.compute.manager [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] [ 656.515662] env[62736]: DEBUG nova.compute.utils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 656.522008] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.760s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.522008] env[62736]: INFO nova.compute.claims [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 656.522008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 5604322cef2049c191c9b5dc03f1b09d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.524362] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Build of instance 4e2f1fc0-102f-4d39-bec2-a2651918a202 was re-scheduled: Binding failed for port 48e4b255-39f8-4691-be1e-141dd9425261, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 656.524976] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 656.525273] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquiring lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 656.525496] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Acquired lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 656.525712] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 656.526170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 30ecec708c094b9e9b1f198bc19c7f9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.533629] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30ecec708c094b9e9b1f198bc19c7f9f [ 656.560652] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5604322cef2049c191c9b5dc03f1b09d [ 656.659882] env[62736]: INFO nova.compute.manager [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] [instance: c97b25ac-527f-404e-a539-1c0f43bcd66b] Took 1.02 seconds to deallocate network for instance. [ 656.661715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 94e031bc20f44eccb295c2a736c0d2ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.715225] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94e031bc20f44eccb295c2a736c0d2ff [ 656.731762] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 656.733922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 2309db89197341cca92fe2ee9b654ba3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.783422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2309db89197341cca92fe2ee9b654ba3 [ 656.793230] env[62736]: DEBUG oslo_vmware.api [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397709, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.246103} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 656.793438] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 656.793611] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 656.793782] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 656.793948] env[62736]: INFO nova.compute.manager [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Took 1.07 seconds to destroy the instance on the hypervisor. [ 656.794184] env[62736]: DEBUG oslo.service.loopingcall [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 656.794356] env[62736]: DEBUG nova.compute.manager [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 656.794445] env[62736]: DEBUG nova.network.neutron [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 656.808418] env[62736]: DEBUG nova.network.neutron [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 656.808916] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 92f9b3dedc684357a479264e0053864b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 656.820446] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92f9b3dedc684357a479264e0053864b [ 657.029525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 7c62cf0d5d674acba3b96d1a3c66b60b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.040750] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c62cf0d5d674acba3b96d1a3c66b60b [ 657.052587] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.152865] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 657.153635] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 8e926efe191c43f0ae401b1137375c5a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.161767] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e926efe191c43f0ae401b1137375c5a [ 657.166131] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 22af80c3581d4334a92ae3fb36e4dc78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.234453] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22af80c3581d4334a92ae3fb36e4dc78 [ 657.254843] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.312157] env[62736]: DEBUG nova.network.neutron [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 657.312157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f6845ffe6763462db444a06966b0d527 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.320393] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6845ffe6763462db444a06966b0d527 [ 657.656437] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Releasing lock "refresh_cache-4e2f1fc0-102f-4d39-bec2-a2651918a202" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 657.656693] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 657.663656] env[62736]: DEBUG nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 657.663656] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 657.676753] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 657.676753] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg bbd6c18e68564c8cae3a7d6495792b58 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.694309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbd6c18e68564c8cae3a7d6495792b58 [ 657.695344] env[62736]: INFO nova.scheduler.client.report [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Deleted allocations for instance c97b25ac-527f-404e-a539-1c0f43bcd66b [ 657.701373] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Expecting reply to msg 2c5c9661d2624d26981087438026b8ed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.736147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c5c9661d2624d26981087438026b8ed [ 657.813912] env[62736]: INFO nova.compute.manager [-] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Took 1.02 seconds to deallocate network for instance. [ 657.818028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg fa64c54da2f841d69124f3c13ffb10ac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 657.850980] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa64c54da2f841d69124f3c13ffb10ac [ 657.991430] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87631956-636a-423e-9940-cb11851c23f4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 657.999426] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5716a3b5-21f9-4ed6-bceb-8695ad31e744 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.032356] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32e18283-f4fc-4a31-a7b4-f84a46c6adb2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.039928] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-094e8bb4-c381-44cc-8d24-9c51ee82d613 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 658.053414] env[62736]: DEBUG nova.compute.provider_tree [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 658.054023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 3f39696c87824e83be95223791f3b035 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.061019] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f39696c87824e83be95223791f3b035 [ 658.189681] env[62736]: DEBUG nova.network.neutron [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 658.190119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 9557c979dd9b4ee096a81cdd903d0535 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.201251] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9557c979dd9b4ee096a81cdd903d0535 [ 658.208802] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7f2ca26b-ac76-4032-b1f1-602304d45f27 tempest-DeleteServersAdminTestJSON-1916310689 tempest-DeleteServersAdminTestJSON-1916310689-project-member] Lock "c97b25ac-527f-404e-a539-1c0f43bcd66b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 92.360s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 658.209405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 706dbeb212b0450db9519b148a75e2d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.223759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 706dbeb212b0450db9519b148a75e2d0 [ 658.320925] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 658.556839] env[62736]: DEBUG nova.scheduler.client.report [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 658.559380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg dec10b0c37964f5d9f3159d520fa4e43 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.575609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dec10b0c37964f5d9f3159d520fa4e43 [ 658.693060] env[62736]: INFO nova.compute.manager [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] [instance: 4e2f1fc0-102f-4d39-bec2-a2651918a202] Took 1.04 seconds to deallocate network for instance. [ 658.694981] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 692a3523ca2943fa9f1600cc6f6118a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.712286] env[62736]: DEBUG nova.compute.manager [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c4512101-c88c-4f3c-bde1-def0f3520101] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 658.714301] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a8824d7062e64b53892bde0591f1a398 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 658.752341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8824d7062e64b53892bde0591f1a398 [ 658.758682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 692a3523ca2943fa9f1600cc6f6118a8 [ 659.062307] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.545s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 659.062879] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 659.064566] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg de0670695f2e427ca169b6ce9d354e4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.065624] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.228s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 659.067398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 3cd1dbed835a4c5c8d60e64c2a2394f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.096950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de0670695f2e427ca169b6ce9d354e4d [ 659.116790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cd1dbed835a4c5c8d60e64c2a2394f0 [ 659.200034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 544fad005d1c4ea9a6f021d9e3f9bdbf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.217179] env[62736]: DEBUG nova.compute.manager [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c4512101-c88c-4f3c-bde1-def0f3520101] Instance disappeared before build. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2439}} [ 659.217598] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg b722a8baa6c442ac94612818ebee6232 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.232076] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b722a8baa6c442ac94612818ebee6232 [ 659.239062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 544fad005d1c4ea9a6f021d9e3f9bdbf [ 659.574294] env[62736]: DEBUG nova.compute.utils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 659.575119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg cdf94f5fdf8e4e269b764d80b559d17a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.576594] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 659.577200] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 659.598427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cdf94f5fdf8e4e269b764d80b559d17a [ 659.642133] env[62736]: DEBUG nova.policy [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '433c8222844b4131b191877d227b9e8c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de518d899df341b9949e44aa38f28a39', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 659.735805] env[62736]: INFO nova.scheduler.client.report [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Deleted allocations for instance 4e2f1fc0-102f-4d39-bec2-a2651918a202 [ 659.745866] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Expecting reply to msg 7b09fb14551f40809d47d42ad70d60ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.747767] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1eeeca-c204-444f-801f-aa11d372fc55 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "c4512101-c88c-4f3c-bde1-def0f3520101" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 74.377s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 659.748293] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg afe93534fbe342ef9c8aaa1949c26e25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 659.757550] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b09fb14551f40809d47d42ad70d60ee [ 659.768390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afe93534fbe342ef9c8aaa1949c26e25 [ 660.048605] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d87015-207a-4cee-a463-006c5296b349 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.063001] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b221edf3-ebfb-4097-aa4f-2f5f35807e93 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.096079] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 660.097758] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg c0081bcacc4b4fda866c9206a91c9057 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.099730] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca12a786-8559-4c94-939b-a8c0cd703ddc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.110389] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2789ee6-70f0-4dfe-b2a2-b615dc508ef5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 660.125297] env[62736]: DEBUG nova.compute.provider_tree [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 660.125830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 27099d7ca94c40a6b8e56611d4de6184 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.134494] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27099d7ca94c40a6b8e56611d4de6184 [ 660.139789] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Successfully created port: 32038c9e-fd2a-4e52-9f5c-d9dc607315cb {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 660.142914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0081bcacc4b4fda866c9206a91c9057 [ 660.252616] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f76d2fd6-064e-4a57-aba4-a3b551967109 tempest-ServerActionsV293TestJSON-1135923703 tempest-ServerActionsV293TestJSON-1135923703-project-member] Lock "4e2f1fc0-102f-4d39-bec2-a2651918a202" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.623s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.254219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg f7ef03d7dc004b33af66bb0095b3011b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.256691] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 660.258392] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 4e8ae43f9f3f4fdfaf54cbd44356754a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.269411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7ef03d7dc004b33af66bb0095b3011b [ 660.296752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e8ae43f9f3f4fdfaf54cbd44356754a [ 660.601897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 0f6ed522d4be48069f00349761dc42ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.630035] env[62736]: DEBUG nova.scheduler.client.report [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 660.631732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 238d9686561a446bb1ea2d2d16f14da2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.645114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f6ed522d4be48069f00349761dc42ca [ 660.647666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 238d9686561a446bb1ea2d2d16f14da2 [ 660.757921] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 660.760224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 28a4bf03891849d6a9f091ff4e8ed763 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 660.778333] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.809160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28a4bf03891849d6a9f091ff4e8ed763 [ 661.105017] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 661.128808] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 661.129045] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 661.129191] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 661.129665] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 661.129665] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 661.129796] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 661.129921] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 661.130093] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 661.130420] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 661.130420] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 661.130600] env[62736]: DEBUG nova.virt.hardware [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 661.131460] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42f726f3-1625-46ee-9aa7-a36dd4cd44b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.134509] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.069s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 661.135104] env[62736]: ERROR nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Traceback (most recent call last): [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.driver.spawn(context, instance, image_meta, [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self._vmops.spawn(context, instance, image_meta, injected_files, [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] vm_ref = self.build_virtual_machine(instance, [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] vif_infos = vmwarevif.get_vif_info(self._session, [ 661.135104] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] for vif in network_info: [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self._sync_wrapper(fn, *args, **kwargs) [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.wait() [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self[:] = self._gt.wait() [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self._exit_event.wait() [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] result = hub.switch() [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 661.135425] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return self.greenlet.switch() [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] result = function(*args, **kwargs) [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] return func(*args, **kwargs) [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise e [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] nwinfo = self.network_api.allocate_for_instance( [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] created_port_ids = self._update_ports_for_instance( [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] with excutils.save_and_reraise_exception(): [ 661.135777] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] self.force_reraise() [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise self.value [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] updated_port = self._update_port( [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] _ensure_no_port_binding_failure(port) [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] raise exception.PortBindingFailed(port_id=port['id']) [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] nova.exception.PortBindingFailed: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. [ 661.136138] env[62736]: ERROR nova.compute.manager [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] [ 661.136442] env[62736]: DEBUG nova.compute.utils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 661.136900] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.555s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 661.138544] env[62736]: INFO nova.compute.claims [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 661.140191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg ab69da07c2c740a19b8ea0d0ed2a4f9e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 661.141861] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Build of instance ce17af27-45c3-461e-8cb7-b9b023d88361 was re-scheduled: Binding failed for port 08f341c7-922c-41cc-8984-e8f0675c276f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 661.142313] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 661.142528] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquiring lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 661.142669] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Acquired lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 661.142847] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 661.143189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg f342dcf602b24d78bcb42f033e0ca26c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 661.147685] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86c942c5-8ae0-48c4-897a-8d0c82caec46 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 661.151811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f342dcf602b24d78bcb42f033e0ca26c [ 661.174662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab69da07c2c740a19b8ea0d0ed2a4f9e [ 661.282199] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 661.409345] env[62736]: ERROR nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 661.409345] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 661.409345] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 661.409345] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 661.409345] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 661.409345] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 661.409345] env[62736]: ERROR nova.compute.manager raise self.value [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 661.409345] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 661.409345] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 661.409345] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 661.409800] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 661.409800] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 661.409800] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 661.409800] env[62736]: ERROR nova.compute.manager [ 661.409800] env[62736]: Traceback (most recent call last): [ 661.409800] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 661.409800] env[62736]: listener.cb(fileno) [ 661.409800] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 661.409800] env[62736]: result = function(*args, **kwargs) [ 661.409800] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 661.409800] env[62736]: return func(*args, **kwargs) [ 661.409800] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 661.409800] env[62736]: raise e [ 661.409800] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 661.409800] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 661.409800] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 661.409800] env[62736]: created_port_ids = self._update_ports_for_instance( [ 661.409800] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 661.409800] env[62736]: with excutils.save_and_reraise_exception(): [ 661.409800] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 661.409800] env[62736]: self.force_reraise() [ 661.409800] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 661.409800] env[62736]: raise self.value [ 661.409800] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 661.409800] env[62736]: updated_port = self._update_port( [ 661.409800] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 661.409800] env[62736]: _ensure_no_port_binding_failure(port) [ 661.409800] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 661.409800] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 661.410565] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 661.410565] env[62736]: Removing descriptor: 16 [ 661.410565] env[62736]: ERROR nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Traceback (most recent call last): [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] yield resources [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.driver.spawn(context, instance, image_meta, [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 661.410565] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] vm_ref = self.build_virtual_machine(instance, [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] vif_infos = vmwarevif.get_vif_info(self._session, [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] for vif in network_info: [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self._sync_wrapper(fn, *args, **kwargs) [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.wait() [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self[:] = self._gt.wait() [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self._exit_event.wait() [ 661.410979] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] result = hub.switch() [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self.greenlet.switch() [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] result = function(*args, **kwargs) [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return func(*args, **kwargs) [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise e [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] nwinfo = self.network_api.allocate_for_instance( [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 661.411355] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] created_port_ids = self._update_ports_for_instance( [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] with excutils.save_and_reraise_exception(): [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.force_reraise() [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise self.value [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] updated_port = self._update_port( [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] _ensure_no_port_binding_failure(port) [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 661.411919] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise exception.PortBindingFailed(port_id=port['id']) [ 661.412701] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 661.412701] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] [ 661.412701] env[62736]: INFO nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Terminating instance [ 661.412701] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquiring lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 661.412701] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquired lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 661.412841] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 661.413257] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg abe39e14728a45beb349cdb30bb1fad8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 661.419403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abe39e14728a45beb349cdb30bb1fad8 [ 661.458575] env[62736]: DEBUG nova.compute.manager [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Received event network-changed-32038c9e-fd2a-4e52-9f5c-d9dc607315cb {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 661.458764] env[62736]: DEBUG nova.compute.manager [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Refreshing instance network info cache due to event network-changed-32038c9e-fd2a-4e52-9f5c-d9dc607315cb. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 661.458942] env[62736]: DEBUG oslo_concurrency.lockutils [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] Acquiring lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 661.646412] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 6861cf90bd7e43a69c8fa238c3f9089a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 661.655129] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6861cf90bd7e43a69c8fa238c3f9089a [ 661.668720] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 661.831828] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 661.832375] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 064badc44956454c8e74c6efe25926b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 661.842465] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 064badc44956454c8e74c6efe25926b9 [ 661.942156] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.015244] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "fd2547e6-9449-4839-8ebb-f02f01a85b8c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.015479] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "fd2547e6-9449-4839-8ebb-f02f01a85b8c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.075388] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.075900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 683e30356cce444e9023bfd003dde410 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.086930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 683e30356cce444e9023bfd003dde410 [ 662.334962] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Releasing lock "refresh_cache-ce17af27-45c3-461e-8cb7-b9b023d88361" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 662.335285] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 662.335500] env[62736]: DEBUG nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 662.335658] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 662.351815] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.352480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg d1a796d40a344eb5b0c660f7b547fa77 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.360485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1a796d40a344eb5b0c660f7b547fa77 [ 662.555569] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ab4079b-d63e-4b8b-b624-cbb0eb1a1302 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.563082] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-540ac1ab-9304-407b-b06f-ec9895523148 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.592622] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Releasing lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 662.593129] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 662.593216] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 662.593706] env[62736]: DEBUG oslo_concurrency.lockutils [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] Acquired lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 662.593843] env[62736]: DEBUG nova.network.neutron [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Refreshing network info cache for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 662.594265] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] Expecting reply to msg 6bfbe4bef97a4dee878477325fc309c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.595179] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e5dc264-8d47-4d19-ba8f-17de968d6a1e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.598465] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d2e0dc5-d4f9-4a23-b491-6d5ea1b16578 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.601437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bfbe4bef97a4dee878477325fc309c1 [ 662.608032] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea5d3166-5d37-4ad2-9332-ab4546e51c7e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.614852] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-467337b5-0f61-44a4-8769-c2fd55c43bec {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 662.634929] env[62736]: DEBUG nova.compute.provider_tree [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 662.635426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 6652f06a8c1f44ab861c39db111e3899 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.640028] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a could not be found. [ 662.640177] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 662.640357] env[62736]: INFO nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Took 0.05 seconds to destroy the instance on the hypervisor. [ 662.640604] env[62736]: DEBUG oslo.service.loopingcall [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 662.641033] env[62736]: DEBUG nova.compute.manager [-] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 662.641130] env[62736]: DEBUG nova.network.neutron [-] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 662.643005] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6652f06a8c1f44ab861c39db111e3899 [ 662.662246] env[62736]: DEBUG nova.network.neutron [-] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.662742] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8437eb1ca4964a168df1f1e87c5960cc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.670211] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8437eb1ca4964a168df1f1e87c5960cc [ 662.855232] env[62736]: DEBUG nova.network.neutron [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.855797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg db4c0708240c4f619b5789660b37c35f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 662.865513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db4c0708240c4f619b5789660b37c35f [ 663.147122] env[62736]: DEBUG nova.scheduler.client.report [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 663.147122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg fe2ca0f20a43409983e6614dc8741a2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.161562] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe2ca0f20a43409983e6614dc8741a2b [ 663.165561] env[62736]: DEBUG nova.network.neutron [-] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.165561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg cbebaf306a7e4ebfb2c1879e90cede9e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.173078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbebaf306a7e4ebfb2c1879e90cede9e [ 663.282966] env[62736]: DEBUG nova.network.neutron [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 663.358275] env[62736]: INFO nova.compute.manager [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] [instance: ce17af27-45c3-461e-8cb7-b9b023d88361] Took 1.02 seconds to deallocate network for instance. [ 663.360096] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 919c65aaa494464fa03980ffa13db660 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.365939] env[62736]: DEBUG nova.network.neutron [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.366393] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] Expecting reply to msg ca9e5515fb284bf0a82277a2fccc0414 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.374953] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca9e5515fb284bf0a82277a2fccc0414 [ 663.414071] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 919c65aaa494464fa03980ffa13db660 [ 663.546970] env[62736]: DEBUG nova.compute.manager [req-eadc665e-fe12-469c-b89b-e7b573ad956b req-bb0b3641-d423-456e-8ba2-c9e0d1501c92 service nova] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Received event network-vif-deleted-32038c9e-fd2a-4e52-9f5c-d9dc607315cb {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 663.649263] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.512s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 663.652730] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 663.652730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg db195fd0030f4cf18191ce15c480d159 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.660369] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.137s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 663.660369] env[62736]: INFO nova.compute.claims [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 663.660369] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 831e307224ef4578918fc36db6e7874d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.669175] env[62736]: INFO nova.compute.manager [-] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Took 1.03 seconds to deallocate network for instance. [ 663.674303] env[62736]: DEBUG nova.compute.claims [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 663.674480] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 663.694518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db195fd0030f4cf18191ce15c480d159 [ 663.701744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 831e307224ef4578918fc36db6e7874d [ 663.864081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg e9c97bf94cb14653b70647a0bd0fa763 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 663.873966] env[62736]: DEBUG oslo_concurrency.lockutils [req-78af07a3-a08a-42ea-9de0-0403d72bcf18 req-c14416da-55e2-4d6a-aaf7-5e8035f17058 service nova] Releasing lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 663.900757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9c97bf94cb14653b70647a0bd0fa763 [ 664.161158] env[62736]: DEBUG nova.compute.utils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 664.161848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg e34fa54de5774cefba2b1753114a465b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 664.162918] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 664.162979] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 664.173973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg e5ba34418058406f8de52a2cb4d76dc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 664.181428] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e34fa54de5774cefba2b1753114a465b [ 664.184688] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5ba34418058406f8de52a2cb4d76dc8 [ 664.252420] env[62736]: DEBUG nova.policy [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '16306c5f0e4d4eabbcce4a9c72edce5f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '532d5738736643d2836c0896012494da', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 664.389939] env[62736]: INFO nova.scheduler.client.report [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Deleted allocations for instance ce17af27-45c3-461e-8cb7-b9b023d88361 [ 664.414743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Expecting reply to msg 1eaafb95a05d44b892b3c353cc08c77f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 664.425883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1eaafb95a05d44b892b3c353cc08c77f [ 664.673848] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 664.675578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg c80b9170c37844458527800aad33116d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 664.721376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c80b9170c37844458527800aad33116d [ 664.748352] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Successfully created port: 0718b236-24c2-4f9c-b72f-49817b897ae7 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 664.914552] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ddb16295-9cca-498a-bd70-98ac0aa624b9 tempest-ServerAddressesNegativeTestJSON-2070091609 tempest-ServerAddressesNegativeTestJSON-2070091609-project-member] Lock "ce17af27-45c3-461e-8cb7-b9b023d88361" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.915296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 06b6837ec62c4d728168c9ed8943870e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 664.925623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06b6837ec62c4d728168c9ed8943870e [ 665.116913] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-442c71f4-1884-48bd-8d73-3a5d967bf414 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.126027] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66a7117d-dc80-4690-a3f4-dac3f7c88b8c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.159468] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33599dee-a3b1-4b9f-9120-f4f226d2afb8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.167084] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c582a7a2-8f7d-4871-9eae-9f77f063a777 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.182371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 5bc1e2cd67e942e08dbfb023a6598a59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 665.183488] env[62736]: DEBUG nova.compute.provider_tree [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 665.183923] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 84e6a7d1fa3441428c62f99acb974db8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 665.190444] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84e6a7d1fa3441428c62f99acb974db8 [ 665.221005] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bc1e2cd67e942e08dbfb023a6598a59 [ 665.417959] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 665.419747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 78f99e3d5e3f40daa37435361065b21c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 665.451223] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78f99e3d5e3f40daa37435361065b21c [ 665.689473] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 665.689473] env[62736]: DEBUG nova.scheduler.client.report [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 665.692718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 913757ea846d4fe3b832f5b6fe326b00 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 665.708272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 913757ea846d4fe3b832f5b6fe326b00 [ 665.720484] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 665.720733] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 665.720887] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 665.721061] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 665.721197] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 665.721336] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 665.721525] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 665.721674] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 665.721835] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 665.721974] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 665.722135] env[62736]: DEBUG nova.virt.hardware [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 665.723264] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aae187f0-eda2-4ec3-ad03-bfac97d88a34 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.737275] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18eabba9-9475-409e-a44c-6b34dab50786 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 665.945211] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 666.195390] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.542s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 666.195928] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 666.197686] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg cfcf7b1bf4a14b4f9800e2f6030f3e50 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 666.198729] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.879s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 666.200582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 0fcebcce4cc0498e92cb5159cf7e9858 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 666.249708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfcf7b1bf4a14b4f9800e2f6030f3e50 [ 666.250984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fcebcce4cc0498e92cb5159cf7e9858 [ 666.344037] env[62736]: DEBUG nova.compute.manager [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Received event network-changed-0718b236-24c2-4f9c-b72f-49817b897ae7 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 666.344333] env[62736]: DEBUG nova.compute.manager [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Refreshing instance network info cache due to event network-changed-0718b236-24c2-4f9c-b72f-49817b897ae7. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 666.344439] env[62736]: DEBUG oslo_concurrency.lockutils [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] Acquiring lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 666.344576] env[62736]: DEBUG oslo_concurrency.lockutils [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] Acquired lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 666.344728] env[62736]: DEBUG nova.network.neutron [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Refreshing network info cache for port 0718b236-24c2-4f9c-b72f-49817b897ae7 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 666.345259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] Expecting reply to msg df8c44c6285d4ee3aa0142566e57ad45 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 666.353439] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df8c44c6285d4ee3aa0142566e57ad45 [ 666.374993] env[62736]: ERROR nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 666.374993] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 666.374993] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 666.374993] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 666.374993] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 666.374993] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 666.374993] env[62736]: ERROR nova.compute.manager raise self.value [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 666.374993] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 666.374993] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 666.374993] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 666.375466] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 666.375466] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 666.375466] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 666.375466] env[62736]: ERROR nova.compute.manager [ 666.375466] env[62736]: Traceback (most recent call last): [ 666.375466] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 666.375466] env[62736]: listener.cb(fileno) [ 666.375466] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 666.375466] env[62736]: result = function(*args, **kwargs) [ 666.375466] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 666.375466] env[62736]: return func(*args, **kwargs) [ 666.375466] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 666.375466] env[62736]: raise e [ 666.375466] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 666.375466] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 666.375466] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 666.375466] env[62736]: created_port_ids = self._update_ports_for_instance( [ 666.375466] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 666.375466] env[62736]: with excutils.save_and_reraise_exception(): [ 666.375466] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 666.375466] env[62736]: self.force_reraise() [ 666.375466] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 666.375466] env[62736]: raise self.value [ 666.375466] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 666.375466] env[62736]: updated_port = self._update_port( [ 666.375466] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 666.375466] env[62736]: _ensure_no_port_binding_failure(port) [ 666.375466] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 666.375466] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 666.376266] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 666.376266] env[62736]: Removing descriptor: 16 [ 666.376266] env[62736]: ERROR nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Traceback (most recent call last): [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] yield resources [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.driver.spawn(context, instance, image_meta, [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self._vmops.spawn(context, instance, image_meta, injected_files, [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 666.376266] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] vm_ref = self.build_virtual_machine(instance, [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] vif_infos = vmwarevif.get_vif_info(self._session, [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] for vif in network_info: [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self._sync_wrapper(fn, *args, **kwargs) [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.wait() [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self[:] = self._gt.wait() [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self._exit_event.wait() [ 666.376597] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] result = hub.switch() [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self.greenlet.switch() [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] result = function(*args, **kwargs) [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return func(*args, **kwargs) [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise e [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] nwinfo = self.network_api.allocate_for_instance( [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 666.376950] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] created_port_ids = self._update_ports_for_instance( [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] with excutils.save_and_reraise_exception(): [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.force_reraise() [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise self.value [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] updated_port = self._update_port( [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] _ensure_no_port_binding_failure(port) [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 666.377301] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise exception.PortBindingFailed(port_id=port['id']) [ 666.377625] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 666.377625] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] [ 666.377625] env[62736]: INFO nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Terminating instance [ 666.378714] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 666.704941] env[62736]: DEBUG nova.compute.utils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 666.704941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 165e2bdbdc364ec6a7de8a5f8610b80c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 666.705092] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 666.710134] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 666.723031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 165e2bdbdc364ec6a7de8a5f8610b80c [ 666.822681] env[62736]: DEBUG nova.policy [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1745c807cc6b4817acff209027fabc8c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4dd6c351f2f24b669fcd5add934e1546', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 666.879463] env[62736]: DEBUG nova.network.neutron [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 667.122508] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Successfully created port: 76838e00-f173-4ff1-94b4-a3c1022b850e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 667.132024] env[62736]: DEBUG nova.network.neutron [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 667.132538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] Expecting reply to msg 12976d09770f40da8e3b797071764cef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.140259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12976d09770f40da8e3b797071764cef [ 667.142568] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2bdc299e-dd00-42f2-b09d-c9f8d94cb4e4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.150568] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c86ba16-1e7e-43f9-abb1-9b36e31359f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.186756] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0929e486-e392-49ec-910c-cc7b4b47a2bf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.198123] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9e953e8-52d9-41e5-90c6-19ee1d85a3ee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 667.213252] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 667.214951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 4bbe036b61404751ae314b97be69f325 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.216834] env[62736]: DEBUG nova.compute.provider_tree [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 667.217363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 9ad0c137a893470c9a90a95f7ed6eb4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.225164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ad0c137a893470c9a90a95f7ed6eb4c [ 667.252021] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bbe036b61404751ae314b97be69f325 [ 667.634714] env[62736]: DEBUG oslo_concurrency.lockutils [req-b9c918c1-34d9-4f9a-ac36-b6228308fe6e req-dbff94a4-f462-4a6a-b8bc-8c0a6121256e service nova] Releasing lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 667.635129] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquired lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 667.635306] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 667.635798] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg a3747cc6e0744c3e852246fa18a15fc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.644391] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3747cc6e0744c3e852246fa18a15fc8 [ 667.722933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg c6c1ad6856404cdaa75572b49ef9cc5f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.728045] env[62736]: DEBUG nova.scheduler.client.report [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 667.728045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg a1487b526d704dda82eb02d3b6a13a8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 667.741802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1487b526d704dda82eb02d3b6a13a8b [ 667.770784] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6c1ad6856404cdaa75572b49ef9cc5f [ 668.162367] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.230161] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 668.232781] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.034s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.233371] env[62736]: ERROR nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Traceback (most recent call last): [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.driver.spawn(context, instance, image_meta, [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self._vmops.spawn(context, instance, image_meta, injected_files, [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] vm_ref = self.build_virtual_machine(instance, [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] vif_infos = vmwarevif.get_vif_info(self._session, [ 668.233371] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] for vif in network_info: [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return self._sync_wrapper(fn, *args, **kwargs) [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.wait() [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self[:] = self._gt.wait() [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return self._exit_event.wait() [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] current.throw(*self._exc) [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 668.233704] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] result = function(*args, **kwargs) [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] return func(*args, **kwargs) [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise e [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] nwinfo = self.network_api.allocate_for_instance( [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] created_port_ids = self._update_ports_for_instance( [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] with excutils.save_and_reraise_exception(): [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] self.force_reraise() [ 668.234025] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise self.value [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] updated_port = self._update_port( [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] _ensure_no_port_binding_failure(port) [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] raise exception.PortBindingFailed(port_id=port['id']) [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] nova.exception.PortBindingFailed: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. [ 668.234342] env[62736]: ERROR nova.compute.manager [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] [ 668.234342] env[62736]: DEBUG nova.compute.utils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 668.240210] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 17.761s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.241139] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 4672b834938a474f8508ce1fcdc4cb97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 668.247084] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Build of instance 9a445d5c-7a5f-4149-b198-306a04b4c040 was re-scheduled: Binding failed for port 012ccbce-db53-42bf-b39c-ede4b3b9b3fc, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 668.247586] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 668.247822] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquiring lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 668.248063] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Acquired lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 668.248245] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 668.248688] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg cafbd932b790417087e18d216e5ef6f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 668.257329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cafbd932b790417087e18d216e5ef6f4 [ 668.265886] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 668.266092] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 668.266241] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 668.266410] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 668.266544] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 668.266677] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 668.266862] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 668.267006] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 668.267155] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 668.267356] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 668.267533] env[62736]: DEBUG nova.virt.hardware [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 668.268613] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-551f7f86-4beb-417c-a6c3-3fa1029c0bbb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.275935] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.276430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg d9d5f6fbf54a4ac18b6bbabd567a028a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 668.278145] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ecd2b634-55c6-4492-8e71-f42585b6fb45 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.292594] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4672b834938a474f8508ce1fcdc4cb97 [ 668.300843] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9d5f6fbf54a4ac18b6bbabd567a028a [ 668.403004] env[62736]: DEBUG nova.compute.manager [req-2e4ff1b0-3090-4405-ac16-ba0c782356b1 req-3065685a-fbc3-4b80-a051-02cd2fbc63b8 service nova] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Received event network-vif-deleted-0718b236-24c2-4f9c-b72f-49817b897ae7 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 668.746852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg f07a45a430b84a268e2aaf3581a31c75 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 668.757472] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f07a45a430b84a268e2aaf3581a31c75 [ 668.783768] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Releasing lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 668.784311] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 668.784500] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 668.784799] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5fc2e31e-afc7-4382-90de-be966aeff760 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.798221] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00b1c8b9-e423-4275-a0d3-d8ee207c9c95 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 668.822174] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 41650a98-fda7-44fd-aca6-d3b94cf17308 could not be found. [ 668.822375] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 668.822586] env[62736]: INFO nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Took 0.04 seconds to destroy the instance on the hypervisor. [ 668.822840] env[62736]: DEBUG oslo.service.loopingcall [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 668.823065] env[62736]: DEBUG nova.compute.manager [-] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 668.823165] env[62736]: DEBUG nova.network.neutron [-] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 668.884355] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.884355] env[62736]: DEBUG nova.network.neutron [-] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.884355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9e6358ac005247e487987479cfebd786 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 668.890404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e6358ac005247e487987479cfebd786 [ 669.036976] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.036976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 050471bfd94d4f58937e1bb5da119be4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.043570] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 050471bfd94d4f58937e1bb5da119be4 [ 669.082080] env[62736]: ERROR nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 669.082080] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 669.082080] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 669.082080] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 669.082080] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 669.082080] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 669.082080] env[62736]: ERROR nova.compute.manager raise self.value [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 669.082080] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 669.082080] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 669.082080] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 669.082488] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 669.082488] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 669.082488] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 669.082488] env[62736]: ERROR nova.compute.manager [ 669.082488] env[62736]: Traceback (most recent call last): [ 669.082488] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 669.082488] env[62736]: listener.cb(fileno) [ 669.082488] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 669.082488] env[62736]: result = function(*args, **kwargs) [ 669.082488] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 669.082488] env[62736]: return func(*args, **kwargs) [ 669.082488] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 669.082488] env[62736]: raise e [ 669.082488] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 669.082488] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 669.082488] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 669.082488] env[62736]: created_port_ids = self._update_ports_for_instance( [ 669.082488] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 669.082488] env[62736]: with excutils.save_and_reraise_exception(): [ 669.082488] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 669.082488] env[62736]: self.force_reraise() [ 669.082488] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 669.082488] env[62736]: raise self.value [ 669.082488] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 669.082488] env[62736]: updated_port = self._update_port( [ 669.082488] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 669.082488] env[62736]: _ensure_no_port_binding_failure(port) [ 669.082488] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 669.082488] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 669.083247] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 669.083247] env[62736]: Removing descriptor: 16 [ 669.083247] env[62736]: ERROR nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Traceback (most recent call last): [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] yield resources [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.driver.spawn(context, instance, image_meta, [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 669.083247] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] vm_ref = self.build_virtual_machine(instance, [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] for vif in network_info: [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self._sync_wrapper(fn, *args, **kwargs) [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.wait() [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self[:] = self._gt.wait() [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self._exit_event.wait() [ 669.083567] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] result = hub.switch() [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self.greenlet.switch() [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] result = function(*args, **kwargs) [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return func(*args, **kwargs) [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise e [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] nwinfo = self.network_api.allocate_for_instance( [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 669.083935] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] created_port_ids = self._update_ports_for_instance( [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] with excutils.save_and_reraise_exception(): [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.force_reraise() [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise self.value [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] updated_port = self._update_port( [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] _ensure_no_port_binding_failure(port) [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 669.084299] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise exception.PortBindingFailed(port_id=port['id']) [ 669.084592] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 669.084592] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] [ 669.084592] env[62736]: INFO nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Terminating instance [ 669.085183] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquiring lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 669.085332] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquired lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 669.085491] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 669.086241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg fc4781f1dd5a4a55815b04b4e3dd7c7f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.093022] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc4781f1dd5a4a55815b04b4e3dd7c7f [ 669.275484] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance b42fe7a4-ec30-441f-b5dc-1f705ee5c881 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.275484] env[62736]: WARNING nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 13247e87-08ab-434e-b091-8b6116280fb7 is not being actively managed by this compute host but has allocations referencing this compute host: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. Skipping heal of allocation because we do not know what to do. [ 669.275484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 7d99e26ac17d46c1b378828d1a6f155e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.285734] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d99e26ac17d46c1b378828d1a6f155e [ 669.385287] env[62736]: DEBUG nova.network.neutron [-] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.385769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8536378efc7c442196899822819083c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.394425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8536378efc7c442196899822819083c6 [ 669.536733] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Releasing lock "refresh_cache-9a445d5c-7a5f-4149-b198-306a04b4c040" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 669.536996] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 669.537178] env[62736]: DEBUG nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 669.537387] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 669.560983] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 669.560983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 90f238d1efa746bfb509034ba0abd8b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.567351] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90f238d1efa746bfb509034ba0abd8b8 [ 669.608759] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 669.779432] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 9a445d5c-7a5f-4149-b198-306a04b4c040 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 669.779432] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance ad008294-52db-40de-95ad-3e0f2d30a462 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.779432] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 38767b46-9a31-4031-981b-424497dc4a34 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.779432] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.779653] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 41650a98-fda7-44fd-aca6-d3b94cf17308 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.779653] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 3e82fa5d-6610-4d3d-972f-4120aca5e8a7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 669.780306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 8dc09fc595b0477d9265d15e8ac49dd0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.790493] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 669.791002] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg db6761d861db4923bc2253aa731b65af in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 669.798769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dc09fc595b0477d9265d15e8ac49dd0 [ 669.803803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db6761d861db4923bc2253aa731b65af [ 669.891200] env[62736]: INFO nova.compute.manager [-] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Took 1.07 seconds to deallocate network for instance. [ 669.893900] env[62736]: DEBUG nova.compute.claims [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 669.894034] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.062707] env[62736]: DEBUG nova.network.neutron [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.063239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 700e1cfbdb464f6fb5232e378085f28b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.072619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 700e1cfbdb464f6fb5232e378085f28b [ 670.393214] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 15215512-da0c-407f-a352-43c051698f2d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 670.393214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 609182513ccb4557a0a64fa5829dd28f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.393214] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Releasing lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 670.393214] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 670.393214] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 670.397868] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f6f0105a-72be-4528-aaef-681a4350c2cb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.397868] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7cb484b2-a34a-426c-8870-6e358be2fd6e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 670.397868] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 609182513ccb4557a0a64fa5829dd28f [ 670.397868] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3e82fa5d-6610-4d3d-972f-4120aca5e8a7 could not be found. [ 670.397868] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 670.397868] env[62736]: INFO nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Took 0.03 seconds to destroy the instance on the hypervisor. [ 670.398828] env[62736]: DEBUG oslo.service.loopingcall [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 670.398828] env[62736]: DEBUG nova.compute.manager [-] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 670.398828] env[62736]: DEBUG nova.network.neutron [-] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 670.398828] env[62736]: DEBUG nova.network.neutron [-] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 670.398828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1cda0e50de49471eb8f303b5ce30a59c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.398828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1cda0e50de49471eb8f303b5ce30a59c [ 670.558716] env[62736]: DEBUG nova.compute.manager [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Received event network-changed-76838e00-f173-4ff1-94b4-a3c1022b850e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 670.558788] env[62736]: DEBUG nova.compute.manager [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Refreshing instance network info cache due to event network-changed-76838e00-f173-4ff1-94b4-a3c1022b850e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 670.558995] env[62736]: DEBUG oslo_concurrency.lockutils [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] Acquiring lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 670.559134] env[62736]: DEBUG oslo_concurrency.lockutils [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] Acquired lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 670.559278] env[62736]: DEBUG nova.network.neutron [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Refreshing network info cache for port 76838e00-f173-4ff1-94b4-a3c1022b850e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 670.559741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] Expecting reply to msg 6f586f767851451cb8cbb1644303797b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.566106] env[62736]: INFO nova.compute.manager [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] [instance: 9a445d5c-7a5f-4149-b198-306a04b4c040] Took 1.03 seconds to deallocate network for instance. [ 670.567643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg f62afbbcc6404092bbf029ef066eddde in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.569458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f586f767851451cb8cbb1644303797b [ 670.613846] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f62afbbcc6404092bbf029ef066eddde [ 670.788962] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance b88d76a9-6a30-4d5f-a5cd-f698e005dab5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 670.789610] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg ef32750f15cd4fd08ab1c2ec6e475e96 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.803876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef32750f15cd4fd08ab1c2ec6e475e96 [ 670.855389] env[62736]: DEBUG nova.network.neutron [-] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.855872] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0e3135e0680948f8b0b9dec6e332ca89 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 670.867108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e3135e0680948f8b0b9dec6e332ca89 [ 671.072791] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg f7d0805121c945eba68297e570d9c9ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 671.081784] env[62736]: DEBUG nova.network.neutron [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 671.117835] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7d0805121c945eba68297e570d9c9ec [ 671.176686] env[62736]: DEBUG nova.network.neutron [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 671.177240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] Expecting reply to msg 86db2e4868cc450b8a58307c8e28d2c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 671.186203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86db2e4868cc450b8a58307c8e28d2c5 [ 671.291743] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance eeb7cdc7-427f-496f-be5d-60151a4bb0f3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 671.292367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg ddb19289c7e949068538907292736296 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 671.303269] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddb19289c7e949068538907292736296 [ 671.360073] env[62736]: INFO nova.compute.manager [-] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Took 1.03 seconds to deallocate network for instance. [ 671.360846] env[62736]: DEBUG nova.compute.claims [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 671.361017] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 671.591843] env[62736]: INFO nova.scheduler.client.report [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Deleted allocations for instance 9a445d5c-7a5f-4149-b198-306a04b4c040 [ 671.597958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Expecting reply to msg 2b430a2560bf4c06b2ba5cdc52fa7bdc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 671.608684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b430a2560bf4c06b2ba5cdc52fa7bdc [ 671.652943] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "5014a870-32fd-4166-9446-c41ed2a4954e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 671.653177] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "5014a870-32fd-4166-9446-c41ed2a4954e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 671.679137] env[62736]: DEBUG oslo_concurrency.lockutils [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] Releasing lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 671.679520] env[62736]: DEBUG nova.compute.manager [req-0dd6b898-0753-442f-9c4f-56609c4e9083 req-068f34fe-0656-4636-b21e-46c6651ac3ef service nova] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Received event network-vif-deleted-76838e00-f173-4ff1-94b4-a3c1022b850e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 671.794725] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 1754ee22-ae95-4439-a6c0-200fd153bad0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 671.795333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg c1cb261dadfd4ec4a59b68b2daeb361f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 671.808806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1cb261dadfd4ec4a59b68b2daeb361f [ 672.100135] env[62736]: DEBUG oslo_concurrency.lockutils [None req-04f976f4-2b10-4abf-ac24-b458e43865bc tempest-ImagesOneServerTestJSON-1857740873 tempest-ImagesOneServerTestJSON-1857740873-project-member] Lock "9a445d5c-7a5f-4149-b198-306a04b4c040" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 94.075s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 672.100769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 300ea7e89c434564a1e870fed29a4e3b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 672.115290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 300ea7e89c434564a1e870fed29a4e3b [ 672.298254] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 71ce0048-90fb-460c-a099-7eff5c092d35 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 672.298885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 1615106e1fb2443a8dfab98ebb1debf0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 672.309327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1615106e1fb2443a8dfab98ebb1debf0 [ 672.602713] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 672.604458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9ee1bdbf1dc14b25b8ff10d7c961feda in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 672.641016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ee1bdbf1dc14b25b8ff10d7c961feda [ 672.802694] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 20d9b891-9c2a-479c-8fd8-93405480dee4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 672.802694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 591cc923e71344768657ed0433abf4cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 672.814181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 591cc923e71344768657ed0433abf4cf [ 673.127387] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 673.305462] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 48a0d4f8-4fab-454b-b065-e552647fba3f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 673.305462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b588a8e97c5a4e6982bf91e0b0ae4330 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 673.315277] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b588a8e97c5a4e6982bf91e0b0ae4330 [ 673.807909] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 459055aa-ae56-480f-b2c6-e45cc05bb50c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 673.807909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 52483430ab024d4c9fb9163f24e902a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 673.823702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52483430ab024d4c9fb9163f24e902a7 [ 674.311731] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 5041b2c1-e7f1-4039-b7c4-1f677918025a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 674.312913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 46cbc55241ef4f549c3fbcdba09ddf15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 674.322325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46cbc55241ef4f549c3fbcdba09ddf15 [ 674.814560] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 896c2790-9107-4d1c-811b-07a87582c0a1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 674.815165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 80651bb150544bb9b81040d99e823073 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 674.825328] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80651bb150544bb9b81040d99e823073 [ 675.317824] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 1174dcea-6ba6-4a2b-9eb4-85e662691c30 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 675.318407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg ff6690ffbb56469f92dc5c4cec8f5e07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 675.329810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff6690ffbb56469f92dc5c4cec8f5e07 [ 675.821470] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 1a5a4c2e-74e6-4a62-adda-c3d89deecf05 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 675.822070] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 64924eb9550e4cee9da6ff78a8b9a683 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 675.832549] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64924eb9550e4cee9da6ff78a8b9a683 [ 676.324923] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 427400bc-f265-40cb-8c1c-ccab7bc94b1c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 676.325526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg f2625bf84b5b4c56a63d1c808f300a92 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 676.335604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f2625bf84b5b4c56a63d1c808f300a92 [ 676.828046] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance d633de23-23ab-4297-b2d6-2702906b31a9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 676.828685] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 328424d4087143e2acd3d2fc9e48d744 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 676.840356] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 328424d4087143e2acd3d2fc9e48d744 [ 677.172561] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquiring lock "b49cde53-0ef7-44a4-b894-e37d1cbf41a7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 677.172794] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "b49cde53-0ef7-44a4-b894-e37d1cbf41a7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 677.332459] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 677.333086] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg e057d19a736146819d67105f2f4a5de0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 677.343341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e057d19a736146819d67105f2f4a5de0 [ 677.835213] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance bbc25649-cfa4-4f5f-ab1e-5f1347780612 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 677.835813] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 7af2e7ee166f44dabcf900d53e564dd1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 677.846222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7af2e7ee166f44dabcf900d53e564dd1 [ 678.338809] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 678.339398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 15399bd2f688490d811fc170b44ff73e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 678.349944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15399bd2f688490d811fc170b44ff73e [ 678.842023] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 3135faf8-b918-42f5-bcaa-c23a19166bb3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 678.842743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg c31eb3d3f66946478dea5d0ad999ac1c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 678.852894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c31eb3d3f66946478dea5d0ad999ac1c [ 679.345658] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 97927074-a676-4ebd-aa7a-fe92e3ce1be6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 679.346253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 749823cfa64e44149f4721ebd205dc01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 679.357694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 749823cfa64e44149f4721ebd205dc01 [ 679.849229] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 61f31838-2fd1-4804-82b7-56b3a08c3ff5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 679.849843] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg bfeea6067f7344f3993f776cc88ed4f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 679.860024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfeea6067f7344f3993f776cc88ed4f7 [ 680.352809] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 680.353395] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 2ce21c9b9174404db4a6fb19ba57ac94 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 680.364136] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ce21c9b9174404db4a6fb19ba57ac94 [ 680.855878] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance fd2547e6-9449-4839-8ebb-f02f01a85b8c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 680.856187] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 6 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 680.856330] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=6GB total_vcpus=48 used_vcpus=6 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 681.214300] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7950096e-8630-41c8-bd18-bab22dde8be5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.221588] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5403de41-3349-4b0a-a1e9-367e2b2d60b7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.251196] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79095a54-3e98-4ba4-9171-0eecb08a40bc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.258203] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52c788f9-2c51-49e4-b49a-658050d96688 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 681.271156] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 681.271614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg d75ab143f20f43a3b6fed322c9cd77b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 681.278632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d75ab143f20f43a3b6fed322c9cd77b8 [ 681.773935] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 681.776564] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 105cbd4c01ea4529af668bcda25e169c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 681.788474] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 105cbd4c01ea4529af668bcda25e169c [ 682.280012] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 682.280012] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 14.044s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.280012] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.636s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.281519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 74b83a47110e4b199ccb6091f679257b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 682.316829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74b83a47110e4b199ccb6091f679257b [ 683.127010] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-029e4fdd-5f14-49d7-b0f9-b908091fdcbf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.135052] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0eda07e5-0feb-426b-bceb-0574f7503e7b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.166408] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4108b6a7-109d-4291-bf48-c7e8352faa18 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.174458] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97eede86-d040-4342-a789-9c19c66f9be8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.190809] env[62736]: DEBUG nova.compute.provider_tree [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 683.191318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 11f0c0de3d1340a1be5250c37049d7c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 683.198237] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11f0c0de3d1340a1be5250c37049d7c4 [ 683.694366] env[62736]: DEBUG nova.scheduler.client.report [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 683.696817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 56c537f3910a48efb8a91c3ce156314d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 683.707966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56c537f3910a48efb8a91c3ce156314d [ 684.199584] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.920s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 684.200286] env[62736]: ERROR nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Traceback (most recent call last): [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.driver.spawn(context, instance, image_meta, [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self._vmops.spawn(context, instance, image_meta, injected_files, [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] vm_ref = self.build_virtual_machine(instance, [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] vif_infos = vmwarevif.get_vif_info(self._session, [ 684.200286] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] for vif in network_info: [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self._sync_wrapper(fn, *args, **kwargs) [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.wait() [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self[:] = self._gt.wait() [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self._exit_event.wait() [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] result = hub.switch() [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 684.200691] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return self.greenlet.switch() [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] result = function(*args, **kwargs) [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] return func(*args, **kwargs) [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise e [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] nwinfo = self.network_api.allocate_for_instance( [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] created_port_ids = self._update_ports_for_instance( [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] with excutils.save_and_reraise_exception(): [ 684.201086] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] self.force_reraise() [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise self.value [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] updated_port = self._update_port( [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] _ensure_no_port_binding_failure(port) [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] raise exception.PortBindingFailed(port_id=port['id']) [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] nova.exception.PortBindingFailed: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. [ 684.201455] env[62736]: ERROR nova.compute.manager [instance: ad008294-52db-40de-95ad-3e0f2d30a462] [ 684.201851] env[62736]: DEBUG nova.compute.utils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 684.202310] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 30.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 684.202489] env[62736]: DEBUG nova.objects.instance [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62736) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 684.203989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg d8dbad930a1545a5aaddaaadaea48697 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 684.206148] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Build of instance ad008294-52db-40de-95ad-3e0f2d30a462 was re-scheduled: Binding failed for port 8a271f17-c894-4132-ac3d-e76ca3e8fbfa, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 684.206600] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 684.206874] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 684.206959] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 684.207108] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 684.207466] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg c2a25fb8c9f64a308904756c38a242c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 684.214007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2a25fb8c9f64a308904756c38a242c4 [ 684.245643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8dbad930a1545a5aaddaaadaea48697 [ 684.708964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg abce6ff377bd4ff68698b40e201e6cce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 684.717818] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abce6ff377bd4ff68698b40e201e6cce [ 684.728486] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 684.798012] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 684.798527] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg ac2e5f86fd884edfa03ba254fa294da3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 684.807024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac2e5f86fd884edfa03ba254fa294da3 [ 685.212149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.010s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.212548] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0b3f5c14-eda3-4bde-8327-dd5c1fa471e4 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg cdceeb4224164ca8a5fe7dc156c27647 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 685.213460] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.599s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.215183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 3263a73ddc0c4bb9b879cc3efaf53b56 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 685.231927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cdceeb4224164ca8a5fe7dc156c27647 [ 685.259452] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3263a73ddc0c4bb9b879cc3efaf53b56 [ 685.300953] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-ad008294-52db-40de-95ad-3e0f2d30a462" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 685.301203] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 685.301382] env[62736]: DEBUG nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 685.301542] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 685.318914] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 685.319585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 5c4fe2f4e6584c00a6187c6a9ce86901 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 685.326930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c4fe2f4e6584c00a6187c6a9ce86901 [ 685.821956] env[62736]: DEBUG nova.network.neutron [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 685.822475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg b8614cf8ddd14aa28f2766cc24b594a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 685.831459] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8614cf8ddd14aa28f2766cc24b594a9 [ 686.022626] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57e8c706-3efd-4309-85e8-fc37e543f081 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.030059] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-24781fe7-69de-4f1c-8409-183a225eb51d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.059857] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d52f4535-c57f-46dd-aa63-fe047e785343 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.066728] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32649c75-4a09-43d3-b282-609a3e1533ea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.079438] env[62736]: DEBUG nova.compute.provider_tree [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 686.079964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 8f481e03b8c4467ab348272f0d2932ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 686.086653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f481e03b8c4467ab348272f0d2932ab [ 686.324538] env[62736]: INFO nova.compute.manager [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: ad008294-52db-40de-95ad-3e0f2d30a462] Took 1.02 seconds to deallocate network for instance. [ 686.326384] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 6f6a46f9860a4710b11aeac9a90f81b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 686.358830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f6a46f9860a4710b11aeac9a90f81b4 [ 686.582337] env[62736]: DEBUG nova.scheduler.client.report [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 686.584712] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg a382ae7dbb9a4070862d654ba200fc09 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 686.595522] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a382ae7dbb9a4070862d654ba200fc09 [ 686.831721] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg e23dcded1e6f48a8949877644bde46a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 686.860657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e23dcded1e6f48a8949877644bde46a6 [ 687.087399] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.874s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 687.088075] env[62736]: ERROR nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] Traceback (most recent call last): [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.driver.spawn(context, instance, image_meta, [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self._vmops.spawn(context, instance, image_meta, injected_files, [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] vm_ref = self.build_virtual_machine(instance, [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] vif_infos = vmwarevif.get_vif_info(self._session, [ 687.088075] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] for vif in network_info: [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return self._sync_wrapper(fn, *args, **kwargs) [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.wait() [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self[:] = self._gt.wait() [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return self._exit_event.wait() [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] current.throw(*self._exc) [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 687.088467] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] result = function(*args, **kwargs) [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] return func(*args, **kwargs) [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise e [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] nwinfo = self.network_api.allocate_for_instance( [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] created_port_ids = self._update_ports_for_instance( [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] with excutils.save_and_reraise_exception(): [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] self.force_reraise() [ 687.088894] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise self.value [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] updated_port = self._update_port( [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] _ensure_no_port_binding_failure(port) [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] raise exception.PortBindingFailed(port_id=port['id']) [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] nova.exception.PortBindingFailed: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. [ 687.089268] env[62736]: ERROR nova.compute.manager [instance: 38767b46-9a31-4031-981b-424497dc4a34] [ 687.089268] env[62736]: DEBUG nova.compute.utils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 687.090023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.836s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 687.091589] env[62736]: INFO nova.compute.claims [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 687.093234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg a08b9f9dc40342cbb723a7a9ede6db2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.094443] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Build of instance 38767b46-9a31-4031-981b-424497dc4a34 was re-scheduled: Binding failed for port 04b56e77-5706-4977-9ead-92b48295fcb5, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 687.094850] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 687.095069] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquiring lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 687.095212] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Acquired lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 687.095366] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 687.095711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 17292a7ab4d745b7bc54d862fa0a92f5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.102531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17292a7ab4d745b7bc54d862fa0a92f5 [ 687.130673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a08b9f9dc40342cbb723a7a9ede6db2b [ 687.357472] env[62736]: INFO nova.scheduler.client.report [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Deleted allocations for instance ad008294-52db-40de-95ad-3e0f2d30a462 [ 687.363671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 99dfb5d584e948519df419a1a218d0a5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.373747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99dfb5d584e948519df419a1a218d0a5 [ 687.599354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 9786ebc54df04fbc80daad2db51d9ad6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.607047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9786ebc54df04fbc80daad2db51d9ad6 [ 687.617676] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 687.693068] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 687.693570] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg f2288049f8eb4e82ab7f19a1841529ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.701600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f2288049f8eb4e82ab7f19a1841529ce [ 687.865724] env[62736]: DEBUG oslo_concurrency.lockutils [None req-65ab93c4-5104-4f28-8ec5-d058d5e89d95 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "ad008294-52db-40de-95ad-3e0f2d30a462" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 108.944s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 687.866358] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 176f5199032a471db5c4d71b3eff0735 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 687.877762] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 176f5199032a471db5c4d71b3eff0735 [ 688.196468] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Releasing lock "refresh_cache-38767b46-9a31-4031-981b-424497dc4a34" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 688.196638] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 688.197319] env[62736]: DEBUG nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 688.197319] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 688.212919] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 688.213575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg d69e2e26ff0a428faf836107f670667b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 688.223028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d69e2e26ff0a428faf836107f670667b [ 688.368772] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 688.370552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 2a86a24762294294a5c36bbfac804eca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 688.403174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a86a24762294294a5c36bbfac804eca [ 688.489067] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d09ea235-b133-47f4-aa9b-b6718f2fe26f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.499226] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54984375-4509-47bd-ba0a-f1287bc96718 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.529004] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2d8d5df-fbdc-476c-981a-104f5623f0af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.536111] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2de955f4-3a50-4229-9937-8538e2e5a384 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.548692] env[62736]: DEBUG nova.compute.provider_tree [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 688.549178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 03e55e34532c4f3a8d6797d9563dbe8e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 688.562890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03e55e34532c4f3a8d6797d9563dbe8e [ 688.715699] env[62736]: DEBUG nova.network.neutron [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 688.716270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 535d0359cbd842d0ba65a63814bde9e4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 688.727744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 535d0359cbd842d0ba65a63814bde9e4 [ 688.890468] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 689.052359] env[62736]: DEBUG nova.scheduler.client.report [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 689.054634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 7ddefece17cc4712a57b48d9429b230c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.064772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ddefece17cc4712a57b48d9429b230c [ 689.218684] env[62736]: INFO nova.compute.manager [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] [instance: 38767b46-9a31-4031-981b-424497dc4a34] Took 1.02 seconds to deallocate network for instance. [ 689.220460] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 168ae37a43064baf8af93714b74db3e2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.259890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 168ae37a43064baf8af93714b74db3e2 [ 689.556904] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.467s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 689.557404] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 689.559177] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 4566816814cf47b884c51ebd4be16405 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.560275] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 31.240s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 689.560504] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 689.562432] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 28.784s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 689.563767] env[62736]: INFO nova.compute.claims [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 689.565228] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 622986a0b38444b3a07a79b974f46f9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.587264] env[62736]: INFO nova.scheduler.client.report [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted allocations for instance 13247e87-08ab-434e-b091-8b6116280fb7 [ 689.589202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg d3664e51033a4cd0ab35292ab8b4c1c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.595297] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4566816814cf47b884c51ebd4be16405 [ 689.600568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 622986a0b38444b3a07a79b974f46f9a [ 689.639748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3664e51033a4cd0ab35292ab8b4c1c1 [ 689.725200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg deb286394b684efb8ee1a0ea7d23a9c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.757870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg deb286394b684efb8ee1a0ea7d23a9c9 [ 689.939485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 6ab05b66f9184da68940eaba2bde3702 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 689.951671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ab05b66f9184da68940eaba2bde3702 [ 690.069215] env[62736]: DEBUG nova.compute.utils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 690.069924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg abf7260dccac42339bfe0028abfe5d99 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.072287] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 528bf0f59e9b40a692cc5b586698564c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.073218] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 690.073386] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 690.080630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 528bf0f59e9b40a692cc5b586698564c [ 690.084794] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abf7260dccac42339bfe0028abfe5d99 [ 690.093389] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "13247e87-08ab-434e-b091-8b6116280fb7" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 35.452s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.093743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfe1311a-9ebd-406b-b870-6e1fcef246fd tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8d3ec890cf8f49999454f526059b4541 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.105785] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d3ec890cf8f49999454f526059b4541 [ 690.246017] env[62736]: INFO nova.scheduler.client.report [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Deleted allocations for instance 38767b46-9a31-4031-981b-424497dc4a34 [ 690.254276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Expecting reply to msg 4a7a258b64864a198f583c3790d3d396 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.265394] env[62736]: DEBUG nova.policy [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b5619794672b4d01a88e58fb3c17e65c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c43001da6eb843efbcbee6303e4a6d00', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 690.277493] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a7a258b64864a198f583c3790d3d396 [ 690.443071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.443071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.443071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.443071] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.443341] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.444913] env[62736]: INFO nova.compute.manager [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Terminating instance [ 690.447024] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "refresh_cache-b42fe7a4-ec30-441f-b5dc-1f705ee5c881" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.447170] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquired lock "refresh_cache-b42fe7a4-ec30-441f-b5dc-1f705ee5c881" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 690.447326] env[62736]: DEBUG nova.network.neutron [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 690.447732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 8079791856dc4fbf945aa50dd82c9c01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.454101] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8079791856dc4fbf945aa50dd82c9c01 [ 690.574550] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 690.576426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg e56b09a73f434cf98db23c873adfe38a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.591155] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Successfully created port: 56565dec-4584-4838-afdb-14c02b5da697 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 690.604085] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "25e906f6-ec98-4c99-83c7-43f94c5790dd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.604379] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "25e906f6-ec98-4c99-83c7-43f94c5790dd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.622489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e56b09a73f434cf98db23c873adfe38a [ 690.769702] env[62736]: DEBUG oslo_concurrency.lockutils [None req-0ec7a21f-d5ff-4ccf-8039-bee024fa795d tempest-InstanceActionsV221TestJSON-646378659 tempest-InstanceActionsV221TestJSON-646378659-project-member] Lock "38767b46-9a31-4031-981b-424497dc4a34" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 109.880s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.770468] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 07ef4badec89404ca4b916014caf606b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 690.784489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07ef4badec89404ca4b916014caf606b [ 690.964669] env[62736]: DEBUG nova.network.neutron [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 690.984231] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59d0c742-d70f-4e16-9049-55880acd5ab9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 690.991621] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73d2c7dd-6b1b-4391-b085-ec1dfb8ecb30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.023291] env[62736]: DEBUG nova.network.neutron [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.023982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg d5987de6c8dc4d6cb2abeeab932a9338 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.025391] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-632c179d-bfb9-421c-85d0-538a6f56072c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.033506] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84bc1056-b91c-43d5-a08e-62cadf50bdf5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.037814] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5987de6c8dc4d6cb2abeeab932a9338 [ 691.050796] env[62736]: DEBUG nova.compute.provider_tree [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 691.051396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 91df982e87604b20a67375f5dae97cb2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.059272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91df982e87604b20a67375f5dae97cb2 [ 691.084095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 0ac5bd39c9f84519be1eb8408614255a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.135766] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ac5bd39c9f84519be1eb8408614255a [ 691.272954] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 691.274787] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg d6e271627ab44b789ddc99d41c3a2687 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.315481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6e271627ab44b789ddc99d41c3a2687 [ 691.391064] env[62736]: DEBUG nova.compute.manager [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Received event network-changed-56565dec-4584-4838-afdb-14c02b5da697 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 691.391255] env[62736]: DEBUG nova.compute.manager [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Refreshing instance network info cache due to event network-changed-56565dec-4584-4838-afdb-14c02b5da697. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 691.391472] env[62736]: DEBUG oslo_concurrency.lockutils [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] Acquiring lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 691.391605] env[62736]: DEBUG oslo_concurrency.lockutils [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] Acquired lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 691.391756] env[62736]: DEBUG nova.network.neutron [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Refreshing network info cache for port 56565dec-4584-4838-afdb-14c02b5da697 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 691.392186] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] Expecting reply to msg d996b58e9a5d43acb56a6f4bbc3808dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.399608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d996b58e9a5d43acb56a6f4bbc3808dd [ 691.479034] env[62736]: ERROR nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 691.479034] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 691.479034] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 691.479034] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 691.479034] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 691.479034] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 691.479034] env[62736]: ERROR nova.compute.manager raise self.value [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 691.479034] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 691.479034] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 691.479034] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 691.479647] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 691.479647] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 691.479647] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 691.479647] env[62736]: ERROR nova.compute.manager [ 691.480027] env[62736]: Traceback (most recent call last): [ 691.480142] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 691.480142] env[62736]: listener.cb(fileno) [ 691.480222] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 691.480222] env[62736]: result = function(*args, **kwargs) [ 691.480297] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 691.480297] env[62736]: return func(*args, **kwargs) [ 691.480388] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 691.480388] env[62736]: raise e [ 691.480466] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 691.480466] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 691.480536] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 691.480536] env[62736]: created_port_ids = self._update_ports_for_instance( [ 691.480615] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 691.480615] env[62736]: with excutils.save_and_reraise_exception(): [ 691.480685] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 691.480685] env[62736]: self.force_reraise() [ 691.480757] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 691.480757] env[62736]: raise self.value [ 691.480828] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 691.480828] env[62736]: updated_port = self._update_port( [ 691.480903] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 691.480903] env[62736]: _ensure_no_port_binding_failure(port) [ 691.480972] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 691.480972] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 691.481053] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 691.481103] env[62736]: Removing descriptor: 16 [ 691.529444] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Releasing lock "refresh_cache-b42fe7a4-ec30-441f-b5dc-1f705ee5c881" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 691.530011] env[62736]: DEBUG nova.compute.manager [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 691.530357] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 691.532275] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d5535ec-af23-4a74-8a89-fc3c04b38e3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.541365] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 691.541728] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-e59b0059-99d2-4e96-aa1b-35f03aa192a6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.547506] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 691.547506] env[62736]: value = "task-397711" [ 691.547506] env[62736]: _type = "Task" [ 691.547506] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 691.556443] env[62736]: DEBUG nova.scheduler.client.report [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 691.559090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 9c81c03e7a9945d98dfe84161fa16162 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 691.560187] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397711, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 691.574732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c81c03e7a9945d98dfe84161fa16162 [ 691.587978] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 691.615856] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 691.616535] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 691.616920] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 691.617374] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 691.617646] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 691.617901] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 691.618233] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 691.618534] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 691.618827] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 691.619110] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 691.619437] env[62736]: DEBUG nova.virt.hardware [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 691.620546] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6b6e843-4f26-450c-a648-bf4a0e1700c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.629509] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a1a1f01-3dc6-4486-ac8d-ddb58f82256a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 691.646971] env[62736]: ERROR nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] Traceback (most recent call last): [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] yield resources [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.driver.spawn(context, instance, image_meta, [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] vm_ref = self.build_virtual_machine(instance, [ 691.646971] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] vif_infos = vmwarevif.get_vif_info(self._session, [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] for vif in network_info: [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return self._sync_wrapper(fn, *args, **kwargs) [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.wait() [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self[:] = self._gt.wait() [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return self._exit_event.wait() [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 691.647383] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] current.throw(*self._exc) [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] result = function(*args, **kwargs) [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return func(*args, **kwargs) [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise e [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] nwinfo = self.network_api.allocate_for_instance( [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] created_port_ids = self._update_ports_for_instance( [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] with excutils.save_and_reraise_exception(): [ 691.647895] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.force_reraise() [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise self.value [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] updated_port = self._update_port( [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] _ensure_no_port_binding_failure(port) [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise exception.PortBindingFailed(port_id=port['id']) [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 691.648334] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] [ 691.649118] env[62736]: INFO nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Terminating instance [ 691.651401] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquiring lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 691.797547] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.912291] env[62736]: DEBUG nova.network.neutron [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.058263] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397711, 'name': PowerOffVM_Task, 'duration_secs': 0.251016} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 692.059530] env[62736]: DEBUG nova.network.neutron [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 692.060197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] Expecting reply to msg 143ce70167f945f4ad1a6ce39a4602c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.061278] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 692.061579] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 692.062382] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.500s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 692.062939] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 692.065035] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg d7e2f6e1d3674e0db423312378c46124 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.066193] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0ab8e858-b164-4336-9bb9-5cb6a6483984 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.067983] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.786s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 692.069469] env[62736]: INFO nova.compute.claims [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 692.071029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 22588dbef85d471091487bfdadb9ecef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.072939] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 143ce70167f945f4ad1a6ce39a4602c8 [ 692.093702] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 692.094418] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Deleting contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 692.094756] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleting the datastore file [datastore2] b42fe7a4-ec30-441f-b5dc-1f705ee5c881 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 692.095152] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a2553b88-86fc-448d-aa79-56725a4bb827 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 692.103130] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for the task: (returnval){ [ 692.103130] env[62736]: value = "task-397713" [ 692.103130] env[62736]: _type = "Task" [ 692.103130] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 692.107222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7e2f6e1d3674e0db423312378c46124 [ 692.112948] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397713, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 692.123841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22588dbef85d471091487bfdadb9ecef [ 692.567453] env[62736]: DEBUG oslo_concurrency.lockutils [req-c3897940-1299-455a-8ea2-a00d21f77737 req-2af56a53-1ae6-433e-a291-70c23dfbe554 service nova] Releasing lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 692.567882] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquired lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 692.568098] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 692.568675] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 91e78f0559e64169a7841df57c473f1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.577764] env[62736]: DEBUG nova.compute.utils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 692.578362] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg f6279eb889494529bbdc3ba69b79f3c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.580315] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 1e6bf4092b7f4dbcb15b4d92815d7a02 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.581218] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91e78f0559e64169a7841df57c473f1e [ 692.581609] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 692.581802] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 692.598626] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e6bf4092b7f4dbcb15b4d92815d7a02 [ 692.599711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6279eb889494529bbdc3ba69b79f3c7 [ 692.612792] env[62736]: DEBUG oslo_vmware.api [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Task: {'id': task-397713, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.120802} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 692.613155] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 692.613541] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Deleted contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 692.613821] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 692.614110] env[62736]: INFO nova.compute.manager [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Took 1.08 seconds to destroy the instance on the hypervisor. [ 692.614523] env[62736]: DEBUG oslo.service.loopingcall [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 692.614805] env[62736]: DEBUG nova.compute.manager [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 692.614983] env[62736]: DEBUG nova.network.neutron [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 692.641289] env[62736]: DEBUG nova.network.neutron [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.642061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3a8b09bef20b47688cfc45a8f3407010 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 692.652270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a8b09bef20b47688cfc45a8f3407010 [ 692.733138] env[62736]: DEBUG nova.policy [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b6684e39ea434b0cbd9623d38b1e68e0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3bca108d42ad437da93a84c160dcade4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 693.085681] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 693.087506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg bda7ab413a4644f4b994fa396efd8bf7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.093590] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 693.127031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bda7ab413a4644f4b994fa396efd8bf7 [ 693.144343] env[62736]: DEBUG nova.network.neutron [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 693.144789] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg cca64b6e1d32476fbe88588ab28f97ed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.154687] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cca64b6e1d32476fbe88588ab28f97ed [ 693.160401] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Successfully created port: ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 693.365343] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 693.365952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg c38eefd8db894df2af7af325c6551563 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.380938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c38eefd8db894df2af7af325c6551563 [ 693.464705] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-025e0c69-ed06-4578-9f31-3bd7d88f6d86 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.473652] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c89098d8-784f-401c-adc2-a918a023cab9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.505158] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99996cae-f090-4b2b-8345-8a0afcacb5c8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.512805] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c9fc82e-e2b9-420a-a6d5-83598752c49b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.519671] env[62736]: DEBUG nova.compute.manager [req-0cf9aefd-cc8d-47bf-827a-49504cf78e53 req-72e5946d-6653-41ab-ab55-347f862a0fc7 service nova] [instance: 15215512-da0c-407f-a352-43c051698f2d] Received event network-vif-deleted-56565dec-4584-4838-afdb-14c02b5da697 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 693.527810] env[62736]: DEBUG nova.compute.provider_tree [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 693.528440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 9ce92a6fdc2345cbb9f8e5163db0b418 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.534913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ce92a6fdc2345cbb9f8e5163db0b418 [ 693.593282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 4beba87b6407498b8e89c11dabb2df23 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.628176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4beba87b6407498b8e89c11dabb2df23 [ 693.646625] env[62736]: INFO nova.compute.manager [-] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Took 1.03 seconds to deallocate network for instance. [ 693.650695] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg c33ae603feca43d4a6d98be4429f8cf4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.680958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c33ae603feca43d4a6d98be4429f8cf4 [ 693.868804] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Releasing lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 693.869216] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 693.869467] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 693.869765] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c05cfe90-99af-43f5-99d3-e1d1f535f364 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.879433] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e0c51c1-2586-4642-bf04-0ac80a809ebf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 693.900280] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 15215512-da0c-407f-a352-43c051698f2d could not be found. [ 693.900400] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 693.900600] env[62736]: INFO nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 693.900836] env[62736]: DEBUG oslo.service.loopingcall [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 693.901045] env[62736]: DEBUG nova.compute.manager [-] [instance: 15215512-da0c-407f-a352-43c051698f2d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 693.901137] env[62736]: DEBUG nova.network.neutron [-] [instance: 15215512-da0c-407f-a352-43c051698f2d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 693.922401] env[62736]: DEBUG nova.network.neutron [-] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 693.922937] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0cddbbf741ca4ed282e631615164b92e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 693.930418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cddbbf741ca4ed282e631615164b92e [ 694.031885] env[62736]: DEBUG nova.scheduler.client.report [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 694.035054] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 03821ef5ed474bafafb257b5e81a0747 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.056388] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03821ef5ed474bafafb257b5e81a0747 [ 694.096317] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 694.122338] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 694.122660] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 694.122817] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 694.123029] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 694.123245] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 694.123426] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 694.123659] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 694.123861] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 694.124096] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 694.124366] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 694.124578] env[62736]: DEBUG nova.virt.hardware [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 694.125741] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3474bb0-6f28-4088-8178-62481ed33cd7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.136722] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a779e197-0e05-4b88-ab1b-165d5fcd21b1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 694.141359] env[62736]: ERROR nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 694.141359] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.141359] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 694.141359] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 694.141359] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.141359] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.141359] env[62736]: ERROR nova.compute.manager raise self.value [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 694.141359] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 694.141359] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.141359] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 694.141886] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.141886] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 694.141886] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 694.141886] env[62736]: ERROR nova.compute.manager [ 694.141886] env[62736]: Traceback (most recent call last): [ 694.141886] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 694.141886] env[62736]: listener.cb(fileno) [ 694.141886] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 694.141886] env[62736]: result = function(*args, **kwargs) [ 694.141886] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 694.141886] env[62736]: return func(*args, **kwargs) [ 694.141886] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 694.141886] env[62736]: raise e [ 694.141886] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.141886] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 694.141886] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 694.141886] env[62736]: created_port_ids = self._update_ports_for_instance( [ 694.141886] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 694.141886] env[62736]: with excutils.save_and_reraise_exception(): [ 694.141886] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.141886] env[62736]: self.force_reraise() [ 694.141886] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.141886] env[62736]: raise self.value [ 694.141886] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 694.141886] env[62736]: updated_port = self._update_port( [ 694.141886] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.141886] env[62736]: _ensure_no_port_binding_failure(port) [ 694.141886] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.141886] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 694.142746] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 694.142746] env[62736]: Removing descriptor: 16 [ 694.153595] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 694.154576] env[62736]: ERROR nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Traceback (most recent call last): [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] yield resources [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.driver.spawn(context, instance, image_meta, [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] vm_ref = self.build_virtual_machine(instance, [ 694.154576] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] vif_infos = vmwarevif.get_vif_info(self._session, [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] for vif in network_info: [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return self._sync_wrapper(fn, *args, **kwargs) [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.wait() [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self[:] = self._gt.wait() [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return self._exit_event.wait() [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 694.154967] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] current.throw(*self._exc) [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] result = function(*args, **kwargs) [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return func(*args, **kwargs) [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise e [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] nwinfo = self.network_api.allocate_for_instance( [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] created_port_ids = self._update_ports_for_instance( [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] with excutils.save_and_reraise_exception(): [ 694.155349] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.force_reraise() [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise self.value [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] updated_port = self._update_port( [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] _ensure_no_port_binding_failure(port) [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise exception.PortBindingFailed(port_id=port['id']) [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 694.155739] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] [ 694.155739] env[62736]: INFO nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Terminating instance [ 694.157665] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquiring lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 694.157881] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquired lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 694.158259] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 694.158693] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 4e770e63406c4ffc86e8ac8211eaba60 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.166440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e770e63406c4ffc86e8ac8211eaba60 [ 694.424818] env[62736]: DEBUG nova.network.neutron [-] [instance: 15215512-da0c-407f-a352-43c051698f2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.425294] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ad688ef989fd4068b74cde49bf42c210 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.434242] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad688ef989fd4068b74cde49bf42c210 [ 694.538246] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.470s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 694.538781] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 694.540510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 5cf48043852145eeb33b7203f5bc0c49 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.541520] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 30.867s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 694.543147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg b4888d54ab5a48b981f6249435c867d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.573974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5cf48043852145eeb33b7203f5bc0c49 [ 694.582899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4888d54ab5a48b981f6249435c867d9 [ 694.679163] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 694.756167] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.756800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 856eeb8f0a8944eb899542f1ec5a9d87 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 694.765643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 856eeb8f0a8944eb899542f1ec5a9d87 [ 694.930614] env[62736]: INFO nova.compute.manager [-] [instance: 15215512-da0c-407f-a352-43c051698f2d] Took 1.03 seconds to deallocate network for instance. [ 694.932088] env[62736]: DEBUG nova.compute.claims [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 694.932088] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.049420] env[62736]: DEBUG nova.compute.utils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 695.049420] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg a926e26161ac4ba3af59047f7be53942 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.051560] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 695.052193] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 695.061039] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a926e26161ac4ba3af59047f7be53942 [ 695.107074] env[62736]: DEBUG nova.policy [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3363a3996f484f75ad17c48432c970e7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '00550bdf4bcd46fab1979767622472b5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 695.260473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Releasing lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 695.260879] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 695.261057] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 695.261340] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8548c813-4e71-4058-91b4-84f725cc65cb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.271965] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ac7d45c-91a9-453b-a22d-eed6d6c8b438 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.292205] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b88d76a9-6a30-4d5f-a5cd-f698e005dab5 could not be found. [ 695.292421] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 695.292595] env[62736]: INFO nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 695.292834] env[62736]: DEBUG oslo.service.loopingcall [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 695.295091] env[62736]: DEBUG nova.compute.manager [-] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 695.295176] env[62736]: DEBUG nova.network.neutron [-] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 695.312355] env[62736]: DEBUG nova.network.neutron [-] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 695.312815] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 88ce20148f9c4472bc946b217d7cf2a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.319797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88ce20148f9c4472bc946b217d7cf2a8 [ 695.394667] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Successfully created port: a20f45f8-34a8-433b-b086-d64d4a77b648 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 695.426624] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-818d88f8-6597-47e1-9dca-882b59058fc1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.433975] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-352ddefe-adfe-4285-9ad5-4f534f3d8cae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.465933] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d79ac94-f99f-4d56-9d80-c409bfd3a3a9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.473351] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcdc4174-1e0d-424e-8d57-9ff27f2c31d9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 695.486653] env[62736]: DEBUG nova.compute.provider_tree [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 695.487350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 93b9afbb2a2442479679020f5c836783 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.494603] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93b9afbb2a2442479679020f5c836783 [ 695.552189] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 695.554055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg fbff4b01ee96431fa60ab11d918f7aab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.573466] env[62736]: DEBUG nova.compute.manager [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Received event network-changed-ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 695.573663] env[62736]: DEBUG nova.compute.manager [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Refreshing instance network info cache due to event network-changed-ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 695.574238] env[62736]: DEBUG oslo_concurrency.lockutils [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] Acquiring lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 695.574389] env[62736]: DEBUG oslo_concurrency.lockutils [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] Acquired lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 695.574865] env[62736]: DEBUG nova.network.neutron [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Refreshing network info cache for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 695.575290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] Expecting reply to msg 08325a22eb6142b8b3232dd5e062e450 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.582053] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08325a22eb6142b8b3232dd5e062e450 [ 695.588200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbff4b01ee96431fa60ab11d918f7aab [ 695.816454] env[62736]: DEBUG nova.network.neutron [-] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.816454] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg adbb8e88e476497785fc600af74a961d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 695.824741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adbb8e88e476497785fc600af74a961d [ 695.990607] env[62736]: DEBUG nova.scheduler.client.report [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 695.993135] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 51c7651f930448c99f6f6c5c8bb4e464 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.013899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51c7651f930448c99f6f6c5c8bb4e464 [ 696.058331] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 39bc5c9ce84a4a8d9d7b51715f4df49e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.097882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39bc5c9ce84a4a8d9d7b51715f4df49e [ 696.103871] env[62736]: DEBUG nova.network.neutron [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 696.208292] env[62736]: DEBUG nova.network.neutron [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 696.208292] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] Expecting reply to msg db4edab22cb7432087a0075bfb8fd5ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.215848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db4edab22cb7432087a0075bfb8fd5ae [ 696.319207] env[62736]: INFO nova.compute.manager [-] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Took 1.02 seconds to deallocate network for instance. [ 696.320296] env[62736]: DEBUG nova.compute.claims [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 696.320296] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.384310] env[62736]: ERROR nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 696.384310] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 696.384310] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 696.384310] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 696.384310] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 696.384310] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 696.384310] env[62736]: ERROR nova.compute.manager raise self.value [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 696.384310] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 696.384310] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 696.384310] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 696.384797] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 696.384797] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 696.384797] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 696.384797] env[62736]: ERROR nova.compute.manager [ 696.384797] env[62736]: Traceback (most recent call last): [ 696.384797] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 696.384797] env[62736]: listener.cb(fileno) [ 696.384797] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 696.384797] env[62736]: result = function(*args, **kwargs) [ 696.384797] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 696.384797] env[62736]: return func(*args, **kwargs) [ 696.384797] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 696.384797] env[62736]: raise e [ 696.384797] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 696.384797] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 696.384797] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 696.384797] env[62736]: created_port_ids = self._update_ports_for_instance( [ 696.384797] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 696.384797] env[62736]: with excutils.save_and_reraise_exception(): [ 696.384797] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 696.384797] env[62736]: self.force_reraise() [ 696.384797] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 696.384797] env[62736]: raise self.value [ 696.384797] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 696.384797] env[62736]: updated_port = self._update_port( [ 696.384797] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 696.384797] env[62736]: _ensure_no_port_binding_failure(port) [ 696.384797] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 696.384797] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 696.385607] env[62736]: nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 696.385607] env[62736]: Removing descriptor: 16 [ 696.496183] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.954s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.496859] env[62736]: ERROR nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Traceback (most recent call last): [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.driver.spawn(context, instance, image_meta, [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] vm_ref = self.build_virtual_machine(instance, [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] vif_infos = vmwarevif.get_vif_info(self._session, [ 696.496859] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] for vif in network_info: [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self._sync_wrapper(fn, *args, **kwargs) [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.wait() [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self[:] = self._gt.wait() [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self._exit_event.wait() [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] result = hub.switch() [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 696.497233] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return self.greenlet.switch() [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] result = function(*args, **kwargs) [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] return func(*args, **kwargs) [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise e [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] nwinfo = self.network_api.allocate_for_instance( [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] created_port_ids = self._update_ports_for_instance( [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] with excutils.save_and_reraise_exception(): [ 696.497681] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] self.force_reraise() [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise self.value [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] updated_port = self._update_port( [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] _ensure_no_port_binding_failure(port) [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] raise exception.PortBindingFailed(port_id=port['id']) [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] nova.exception.PortBindingFailed: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. [ 696.498056] env[62736]: ERROR nova.compute.manager [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] [ 696.498369] env[62736]: DEBUG nova.compute.utils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 696.498897] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 30.554s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.500383] env[62736]: INFO nova.compute.claims [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 696.502427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 2049d099ced6462786923d0559423343 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.503639] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Build of instance 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a was re-scheduled: Binding failed for port 32038c9e-fd2a-4e52-9f5c-d9dc607315cb, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 696.504082] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 696.504311] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquiring lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 696.504456] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Acquired lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 696.504612] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 696.504961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg ee283515efc54375a91e50967a196a32 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.511676] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee283515efc54375a91e50967a196a32 [ 696.535984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2049d099ced6462786923d0559423343 [ 696.561695] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 696.584287] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 696.584557] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 696.584723] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 696.584902] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 696.585044] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 696.585183] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 696.585379] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 696.585527] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 696.585681] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 696.585833] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 696.585995] env[62736]: DEBUG nova.virt.hardware [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 696.587667] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14684f8b-12b1-4695-bf5b-8cdb0a50bf78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.596023] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11172ba4-61b6-42e2-af5d-9c215afefe5d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 696.615149] env[62736]: ERROR nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Traceback (most recent call last): [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] yield resources [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.driver.spawn(context, instance, image_meta, [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] vm_ref = self.build_virtual_machine(instance, [ 696.615149] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] vif_infos = vmwarevif.get_vif_info(self._session, [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] for vif in network_info: [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return self._sync_wrapper(fn, *args, **kwargs) [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.wait() [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self[:] = self._gt.wait() [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return self._exit_event.wait() [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 696.615508] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] current.throw(*self._exc) [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] result = function(*args, **kwargs) [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return func(*args, **kwargs) [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise e [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] nwinfo = self.network_api.allocate_for_instance( [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] created_port_ids = self._update_ports_for_instance( [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] with excutils.save_and_reraise_exception(): [ 696.616027] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.force_reraise() [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise self.value [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] updated_port = self._update_port( [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] _ensure_no_port_binding_failure(port) [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise exception.PortBindingFailed(port_id=port['id']) [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 696.616582] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] [ 696.616582] env[62736]: INFO nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Terminating instance [ 696.617369] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 696.617569] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquired lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 696.617693] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 696.618097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 616812235f344b1c8ad519ce32f6ee2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 696.625488] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 616812235f344b1c8ad519ce32f6ee2b [ 696.713687] env[62736]: DEBUG oslo_concurrency.lockutils [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] Releasing lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 696.713976] env[62736]: DEBUG nova.compute.manager [req-456239db-266f-4b15-be34-a4293c247cd1 req-f907533f-4028-4bfe-b6cc-f5172bc1dfed service nova] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Received event network-vif-deleted-ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 696.953104] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "99735498-3c85-4a01-bc05-4def014618a8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.953360] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "99735498-3c85-4a01-bc05-4def014618a8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 697.008201] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 58e1d07d0016457290e34a9c52bc7cdf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.024302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58e1d07d0016457290e34a9c52bc7cdf [ 697.029144] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.134538] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.145557] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 697.146080] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 3d15e0107cf649b6a635204566a58b47 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.154601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d15e0107cf649b6a635204566a58b47 [ 697.224179] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 697.224733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 3f9d35729c0a4925a535282a03443279 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.259576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f9d35729c0a4925a535282a03443279 [ 697.458547] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "6c4ac72f-1636-4f4c-928c-0a6fe895ce37" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 697.458792] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "6c4ac72f-1636-4f4c-928c-0a6fe895ce37" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 697.648564] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Releasing lock "refresh_cache-6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 697.649000] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 697.649289] env[62736]: DEBUG nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 697.649609] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 697.675633] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.676658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 552b83186b4e42e0b496ca2b54389dfb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.684050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 552b83186b4e42e0b496ca2b54389dfb [ 697.722079] env[62736]: DEBUG nova.compute.manager [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Received event network-changed-a20f45f8-34a8-433b-b086-d64d4a77b648 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 697.722428] env[62736]: DEBUG nova.compute.manager [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Refreshing instance network info cache due to event network-changed-a20f45f8-34a8-433b-b086-d64d4a77b648. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 697.722854] env[62736]: DEBUG oslo_concurrency.lockutils [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] Acquiring lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 697.739457] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Releasing lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 697.740267] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 697.740688] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 697.743957] env[62736]: DEBUG oslo_concurrency.lockutils [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] Acquired lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 697.744315] env[62736]: DEBUG nova.network.neutron [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Refreshing network info cache for port a20f45f8-34a8-433b-b086-d64d4a77b648 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 697.744941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] Expecting reply to msg a8f39259b5104623a2d34a17e3735450 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.745931] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-d49c6244-b206-4f39-b4ec-dd551c058edb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.754041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8f39259b5104623a2d34a17e3735450 [ 697.757284] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5146fcaa-66d0-4a2c-8114-20e6afbe4c70 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 697.786579] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance eeb7cdc7-427f-496f-be5d-60151a4bb0f3 could not be found. [ 697.787069] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 697.787473] env[62736]: INFO nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Took 0.05 seconds to destroy the instance on the hypervisor. [ 697.788043] env[62736]: DEBUG oslo.service.loopingcall [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 697.788367] env[62736]: DEBUG nova.compute.manager [-] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 697.788624] env[62736]: DEBUG nova.network.neutron [-] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 697.804354] env[62736]: DEBUG nova.network.neutron [-] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 697.805181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c5f798eb26d543d68ef9c13109d0fa80 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 697.811759] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5f798eb26d543d68ef9c13109d0fa80 [ 698.034495] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53c44c76-398f-4f5b-b7b5-5f4e4d684600 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.046154] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d38db740-de8c-41be-b4a3-a3f2be699e86 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.077937] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21544781-251e-4460-933c-13e40c51d40c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.085687] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-181c590f-9369-4ed9-a459-a248f9322c77 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 698.102754] env[62736]: DEBUG nova.compute.provider_tree [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 698.103272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 91c4c23ccf714e2e90a36de73836f4ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.112424] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91c4c23ccf714e2e90a36de73836f4ab [ 698.178935] env[62736]: DEBUG nova.network.neutron [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 698.179614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg de8dcf9c8e18459cb8a26e8f6dd56d4b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.187593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de8dcf9c8e18459cb8a26e8f6dd56d4b [ 698.264995] env[62736]: DEBUG nova.network.neutron [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 698.312077] env[62736]: DEBUG nova.network.neutron [-] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 698.312522] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9ec145c3585649f5b1d192fb4c44230b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.321517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ec145c3585649f5b1d192fb4c44230b [ 698.345056] env[62736]: DEBUG nova.network.neutron [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 698.345541] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] Expecting reply to msg e50d9b45304a4b3b891220058a2086c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.353751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e50d9b45304a4b3b891220058a2086c8 [ 698.605741] env[62736]: DEBUG nova.scheduler.client.report [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 698.608212] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg a1ff9921012b4c39b201982906d96aa1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.619865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1ff9921012b4c39b201982906d96aa1 [ 698.682217] env[62736]: INFO nova.compute.manager [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] [instance: 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a] Took 1.03 seconds to deallocate network for instance. [ 698.683931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 380e23da26c94c368ad6d9a2f625327c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 698.717398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 380e23da26c94c368ad6d9a2f625327c [ 698.814975] env[62736]: INFO nova.compute.manager [-] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Took 1.03 seconds to deallocate network for instance. [ 698.817372] env[62736]: DEBUG nova.compute.claims [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 698.817581] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 698.847400] env[62736]: DEBUG oslo_concurrency.lockutils [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] Releasing lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 698.847714] env[62736]: DEBUG nova.compute.manager [req-8e8bf8fe-13ec-4148-8dab-e09d7ad0ac5f req-e2cfd42f-de49-43a8-ba08-e98f0df67bb9 service nova] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Received event network-vif-deleted-a20f45f8-34a8-433b-b086-d64d4a77b648 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 699.117590] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.612s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.117590] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 699.117590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 238a882f24bd44c6bfe13b546e518bbc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 699.117590] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.220s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.117590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 5da23ff2e87e4fa2b5ba2c371a5031a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 699.157944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5da23ff2e87e4fa2b5ba2c371a5031a0 [ 699.158833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 238a882f24bd44c6bfe13b546e518bbc [ 699.189449] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg c93570d94bf54097ae447cd5d2b26ce9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 699.221278] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c93570d94bf54097ae447cd5d2b26ce9 [ 699.618949] env[62736]: DEBUG nova.compute.utils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 699.619681] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg f07073181c4547baae198a150a24e04c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 699.624051] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 699.624325] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 699.631980] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f07073181c4547baae198a150a24e04c [ 699.664828] env[62736]: DEBUG nova.policy [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3363a3996f484f75ad17c48432c970e7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '00550bdf4bcd46fab1979767622472b5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 699.708042] env[62736]: INFO nova.scheduler.client.report [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Deleted allocations for instance 6480b28b-7ded-46c1-8fa0-6e1abba4bf7a [ 699.715754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Expecting reply to msg 77e73688dece494db3f8cba310594c15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 699.728060] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77e73688dece494db3f8cba310594c15 [ 699.984327] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b5d0c3a-800f-4ab5-97f9-64a8eb192550 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 699.995045] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3488fa0d-5ae2-47b8-97b7-3fa2546f71b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.020792] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7970e4e6-292f-4fee-8346-980fb79d5b48 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.027801] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2926185-c7d9-45ea-930d-5d06d952c671 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 700.033232] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Successfully created port: 9ba330ad-0f59-4b8a-898d-9edc8943371d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 700.043889] env[62736]: DEBUG nova.compute.provider_tree [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 700.044396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg d710f7656f944ec98fc1f1b78671b6f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.052932] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d710f7656f944ec98fc1f1b78671b6f0 [ 700.124745] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 700.126445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 919201302656418f85848e9fa332522e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.158952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 919201302656418f85848e9fa332522e [ 700.217828] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1f17e573-26e4-4b7a-9707-f4de1683531c tempest-ServerPasswordTestJSON-873993815 tempest-ServerPasswordTestJSON-873993815-project-member] Lock "6480b28b-7ded-46c1-8fa0-6e1abba4bf7a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 118.932s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 700.218422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg ab39f4756a39484381e090b8ad4c4d34 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.228094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab39f4756a39484381e090b8ad4c4d34 [ 700.547022] env[62736]: DEBUG nova.scheduler.client.report [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 700.549736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg b00051153f0d4f5c8f1ebb74ef72b414 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.563430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b00051153f0d4f5c8f1ebb74ef72b414 [ 700.642348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg d44ae9a98a374788a11bc13951b32d26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.675530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d44ae9a98a374788a11bc13951b32d26 [ 700.721037] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 700.722767] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg f13df1a02b354b5dae84f0b4890b273b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 700.775990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f13df1a02b354b5dae84f0b4890b273b [ 701.061282] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.943s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 701.061282] env[62736]: ERROR nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Traceback (most recent call last): [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.driver.spawn(context, instance, image_meta, [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self._vmops.spawn(context, instance, image_meta, injected_files, [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 701.061282] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] vm_ref = self.build_virtual_machine(instance, [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] vif_infos = vmwarevif.get_vif_info(self._session, [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] for vif in network_info: [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self._sync_wrapper(fn, *args, **kwargs) [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.wait() [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self[:] = self._gt.wait() [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self._exit_event.wait() [ 701.061889] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] result = hub.switch() [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return self.greenlet.switch() [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] result = function(*args, **kwargs) [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] return func(*args, **kwargs) [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise e [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] nwinfo = self.network_api.allocate_for_instance( [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 701.062449] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] created_port_ids = self._update_ports_for_instance( [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] with excutils.save_and_reraise_exception(): [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] self.force_reraise() [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise self.value [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] updated_port = self._update_port( [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] _ensure_no_port_binding_failure(port) [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 701.062770] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] raise exception.PortBindingFailed(port_id=port['id']) [ 701.063079] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] nova.exception.PortBindingFailed: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. [ 701.063079] env[62736]: ERROR nova.compute.manager [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] [ 701.063079] env[62736]: DEBUG nova.compute.utils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 701.063079] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 29.700s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 701.063079] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 3151d6b4935d413cb81613586db27f16 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 701.063904] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Build of instance 41650a98-fda7-44fd-aca6-d3b94cf17308 was re-scheduled: Binding failed for port 0718b236-24c2-4f9c-b72f-49817b897ae7, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 701.064394] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 701.064649] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 701.064797] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquired lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 701.064961] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 701.065399] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg b07a70faf22e478ba0af3d0b75d98603 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 701.073583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b07a70faf22e478ba0af3d0b75d98603 [ 701.107798] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3151d6b4935d413cb81613586db27f16 [ 701.146441] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 701.173963] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 701.174212] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 701.174366] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 701.174559] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 701.174691] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 701.174835] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 701.175035] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 701.175220] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 701.175616] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 701.175616] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 701.175718] env[62736]: DEBUG nova.virt.hardware [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 701.176928] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68a2eb2f-f9b1-4d72-8131-1b844e9f9a15 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.186201] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f7c2eca-1841-47e7-a98e-051c7e84ebf0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 701.244582] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 701.474858] env[62736]: DEBUG nova.compute.manager [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Received event network-changed-9ba330ad-0f59-4b8a-898d-9edc8943371d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 701.475150] env[62736]: DEBUG nova.compute.manager [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Refreshing instance network info cache due to event network-changed-9ba330ad-0f59-4b8a-898d-9edc8943371d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 701.475456] env[62736]: DEBUG oslo_concurrency.lockutils [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] Acquiring lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 701.475674] env[62736]: DEBUG oslo_concurrency.lockutils [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] Acquired lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 701.475907] env[62736]: DEBUG nova.network.neutron [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Refreshing network info cache for port 9ba330ad-0f59-4b8a-898d-9edc8943371d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 701.476515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] Expecting reply to msg 59cf285426954e048350046d82c3f309 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 701.483871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59cf285426954e048350046d82c3f309 [ 701.606934] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 701.724595] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 701.725378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg f99faa25f4384074995b618c13bd0c7d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 701.743645] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f99faa25f4384074995b618c13bd0c7d [ 701.765428] env[62736]: ERROR nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 701.765428] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 701.765428] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 701.765428] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 701.765428] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 701.765428] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 701.765428] env[62736]: ERROR nova.compute.manager raise self.value [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 701.765428] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 701.765428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 701.765428] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 701.765948] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 701.765948] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 701.765948] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 701.765948] env[62736]: ERROR nova.compute.manager [ 701.765948] env[62736]: Traceback (most recent call last): [ 701.765948] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 701.765948] env[62736]: listener.cb(fileno) [ 701.765948] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 701.765948] env[62736]: result = function(*args, **kwargs) [ 701.765948] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 701.765948] env[62736]: return func(*args, **kwargs) [ 701.765948] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 701.765948] env[62736]: raise e [ 701.765948] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 701.765948] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 701.765948] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 701.765948] env[62736]: created_port_ids = self._update_ports_for_instance( [ 701.765948] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 701.765948] env[62736]: with excutils.save_and_reraise_exception(): [ 701.765948] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 701.765948] env[62736]: self.force_reraise() [ 701.765948] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 701.765948] env[62736]: raise self.value [ 701.765948] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 701.765948] env[62736]: updated_port = self._update_port( [ 701.765948] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 701.765948] env[62736]: _ensure_no_port_binding_failure(port) [ 701.765948] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 701.765948] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 701.766812] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 701.766812] env[62736]: Removing descriptor: 17 [ 701.766812] env[62736]: ERROR nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Traceback (most recent call last): [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] yield resources [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.driver.spawn(context, instance, image_meta, [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 701.766812] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] vm_ref = self.build_virtual_machine(instance, [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] vif_infos = vmwarevif.get_vif_info(self._session, [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] for vif in network_info: [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self._sync_wrapper(fn, *args, **kwargs) [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.wait() [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self[:] = self._gt.wait() [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self._exit_event.wait() [ 701.767164] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] result = hub.switch() [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self.greenlet.switch() [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] result = function(*args, **kwargs) [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return func(*args, **kwargs) [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise e [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] nwinfo = self.network_api.allocate_for_instance( [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 701.767549] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] created_port_ids = self._update_ports_for_instance( [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] with excutils.save_and_reraise_exception(): [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.force_reraise() [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise self.value [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] updated_port = self._update_port( [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] _ensure_no_port_binding_failure(port) [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 701.767911] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise exception.PortBindingFailed(port_id=port['id']) [ 701.768288] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 701.768288] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] [ 701.768288] env[62736]: INFO nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Terminating instance [ 701.768707] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 702.003974] env[62736]: DEBUG nova.network.neutron [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 702.024421] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3a47620-e0e2-4504-be71-14c1f8133650 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.033390] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-200ac2f8-04e4-4376-b4f9-25476177a727 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.068994] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0c13d88-ee71-4b7c-8cc2-a4a8d5d6c4ce {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.078582] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d39e4c0c-1ef1-4520-8204-4d0adcf29b9b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 702.093883] env[62736]: DEBUG nova.compute.provider_tree [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 702.094596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg b8e22abe92e44001b757ad78b39571df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.104909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8e22abe92e44001b757ad78b39571df [ 702.134340] env[62736]: DEBUG nova.network.neutron [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.134340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] Expecting reply to msg 8b902cc5acb44abb91e4d21838868772 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.144977] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b902cc5acb44abb91e4d21838868772 [ 702.227873] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Releasing lock "refresh_cache-41650a98-fda7-44fd-aca6-d3b94cf17308" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 702.228174] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 702.228366] env[62736]: DEBUG nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 702.228707] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 702.244042] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 702.244620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg bc7e61c4afc1432aae4f7580a1cf5a2e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.251518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc7e61c4afc1432aae4f7580a1cf5a2e [ 702.597603] env[62736]: DEBUG nova.scheduler.client.report [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 702.600183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 0269220fd95440b891de0faf91f42f88 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.615078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0269220fd95440b891de0faf91f42f88 [ 702.637150] env[62736]: DEBUG oslo_concurrency.lockutils [req-dca411d0-ebfc-478e-bc0a-ba7e87626a1f req-7252687d-7750-452b-8fe4-1ceb9300722d service nova] Releasing lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 702.637631] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquired lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 702.637882] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 702.638347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 0c6cf608aa3047b6b2eefdd4f5478854 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.645387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c6cf608aa3047b6b2eefdd4f5478854 [ 702.746726] env[62736]: DEBUG nova.network.neutron [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.747260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 35d1cc352cf24e24a1d229ab0bd5cc10 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 702.756130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35d1cc352cf24e24a1d229ab0bd5cc10 [ 703.103471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.043s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.104176] env[62736]: ERROR nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Traceback (most recent call last): [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.driver.spawn(context, instance, image_meta, [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] vm_ref = self.build_virtual_machine(instance, [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 703.104176] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] for vif in network_info: [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self._sync_wrapper(fn, *args, **kwargs) [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.wait() [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self[:] = self._gt.wait() [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self._exit_event.wait() [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] result = hub.switch() [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 703.104560] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return self.greenlet.switch() [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] result = function(*args, **kwargs) [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] return func(*args, **kwargs) [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise e [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] nwinfo = self.network_api.allocate_for_instance( [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] created_port_ids = self._update_ports_for_instance( [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] with excutils.save_and_reraise_exception(): [ 703.104943] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] self.force_reraise() [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise self.value [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] updated_port = self._update_port( [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] _ensure_no_port_binding_failure(port) [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] raise exception.PortBindingFailed(port_id=port['id']) [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] nova.exception.PortBindingFailed: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. [ 703.105322] env[62736]: ERROR nova.compute.manager [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] [ 703.105694] env[62736]: DEBUG nova.compute.utils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 703.106143] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 29.980s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.110183] env[62736]: INFO nova.compute.claims [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 703.112857] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 93732b5ab0ae4663b7a6d74068011449 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.113744] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Build of instance 3e82fa5d-6610-4d3d-972f-4120aca5e8a7 was re-scheduled: Binding failed for port 76838e00-f173-4ff1-94b4-a3c1022b850e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 703.114279] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 703.114512] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquiring lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 703.114658] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Acquired lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 703.114812] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 703.115241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 00dbd8a495c94087a5573eeae28e2ce4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.130130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00dbd8a495c94087a5573eeae28e2ce4 [ 703.148321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93732b5ab0ae4663b7a6d74068011449 [ 703.166152] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.251979] env[62736]: INFO nova.compute.manager [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 41650a98-fda7-44fd-aca6-d3b94cf17308] Took 1.02 seconds to deallocate network for instance. [ 703.251979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 8220828280414e878471447e4b756b4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.255211] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.255211] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 3006989d380d4f7c8a4145782258af26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.263793] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3006989d380d4f7c8a4145782258af26 [ 703.287340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8220828280414e878471447e4b756b4c [ 703.506149] env[62736]: DEBUG nova.compute.manager [req-5f4a4db8-b2ad-4ff2-b72f-6e9fb59b9289 req-ef168eba-902b-4ecc-8d76-86b573ec9975 service nova] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Received event network-vif-deleted-9ba330ad-0f59-4b8a-898d-9edc8943371d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 703.618142] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6822344e077f45589de49158f47d613b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.627162] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6822344e077f45589de49158f47d613b [ 703.663023] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.756863] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 7ee3636c59f546d6996483e22c70dc79 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.758167] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Releasing lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 703.758616] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 703.758845] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 703.759670] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3650e846-f54d-411f-ace6-842365629230 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.762642] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 703.763132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 4e5674a3ba474986a31e14656c203f7c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.772483] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e5674a3ba474986a31e14656c203f7c [ 703.776415] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d5b4d8b-794a-4768-b46d-22647a5afaea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 703.804603] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1754ee22-ae95-4439-a6c0-200fd153bad0 could not be found. [ 703.804603] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 703.804603] env[62736]: INFO nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 703.804603] env[62736]: DEBUG oslo.service.loopingcall [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 703.804603] env[62736]: DEBUG nova.compute.manager [-] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 703.804603] env[62736]: DEBUG nova.network.neutron [-] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 703.816513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ee3636c59f546d6996483e22c70dc79 [ 703.818523] env[62736]: DEBUG nova.network.neutron [-] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 703.819196] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c53506e217794bce9d8c57c705005d9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 703.827429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c53506e217794bce9d8c57c705005d9f [ 704.273023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Releasing lock "refresh_cache-3e82fa5d-6610-4d3d-972f-4120aca5e8a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 704.273259] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 704.273440] env[62736]: DEBUG nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 704.273610] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 704.277794] env[62736]: INFO nova.scheduler.client.report [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Deleted allocations for instance 41650a98-fda7-44fd-aca6-d3b94cf17308 [ 704.283568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 53af81ad43f34dbfb18d5b9de9f26f31 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.302011] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 704.302988] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 2e5f0c29497b484b8dc56ace73c1c386 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.308698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53af81ad43f34dbfb18d5b9de9f26f31 [ 704.311918] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e5f0c29497b484b8dc56ace73c1c386 [ 704.320538] env[62736]: DEBUG nova.network.neutron [-] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.320892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 05d96cbdf14441f7953a1ff7f525472b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.329038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05d96cbdf14441f7953a1ff7f525472b [ 704.480873] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b092918-3e7b-4076-8e85-e12f0b216218 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.488351] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4486d64e-5a13-4857-abab-9bc1677a909a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.519410] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81d9e7d1-d143-4737-a0aa-7d01cfaf8762 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.541402] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9588a9a8-ddb4-463f-91d9-9106fbb4e697 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 704.549873] env[62736]: DEBUG nova.compute.provider_tree [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 704.550516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9415166ddcf54d75bdd2f32b67586a73 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.559063] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9415166ddcf54d75bdd2f32b67586a73 [ 704.789787] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8e565307-45d9-4339-abba-636f1e61428e tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "41650a98-fda7-44fd-aca6-d3b94cf17308" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 122.448s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 704.790529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 90d7d2b5916847cb984367680656eeb3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.799772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90d7d2b5916847cb984367680656eeb3 [ 704.808599] env[62736]: DEBUG nova.network.neutron [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 704.809075] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg af8bdeb3103846e78be3a07383eb4f4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 704.817039] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af8bdeb3103846e78be3a07383eb4f4d [ 704.825191] env[62736]: INFO nova.compute.manager [-] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Took 1.02 seconds to deallocate network for instance. [ 704.827808] env[62736]: DEBUG nova.compute.claims [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 704.827967] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 705.059860] env[62736]: DEBUG nova.scheduler.client.report [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 705.062285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 33f86820302f447ba10347fe422427cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.073460] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33f86820302f447ba10347fe422427cf [ 705.293023] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 705.294791] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg b843b130453c450fa9cbceb33265aa1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.317917] env[62736]: INFO nova.compute.manager [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] [instance: 3e82fa5d-6610-4d3d-972f-4120aca5e8a7] Took 1.04 seconds to deallocate network for instance. [ 705.319620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 94a7feb8e60e422bab41550f9d0b74d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.336010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b843b130453c450fa9cbceb33265aa1d [ 705.354770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94a7feb8e60e422bab41550f9d0b74d5 [ 705.564873] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.459s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 705.565405] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 705.567070] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg bba33189799f4755a35729bea2f91e7b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.568338] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.678s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 705.570162] env[62736]: INFO nova.compute.claims [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 705.571705] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 1f5f3b088cf8425d84916e179e01fa4e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.606853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f5f3b088cf8425d84916e179e01fa4e [ 705.610631] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bba33189799f4755a35729bea2f91e7b [ 705.813713] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 705.827318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 59978c55a0124b7487d12c8f3621af47 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 705.864945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59978c55a0124b7487d12c8f3621af47 [ 706.075136] env[62736]: DEBUG nova.compute.utils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 706.075854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0d57e567944d4dd98a58a259d0d82735 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 706.078105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 4bc077b74e7b4b228cb667debec58275 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 706.086584] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 706.086584] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 706.089657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bc077b74e7b4b228cb667debec58275 [ 706.090565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d57e567944d4dd98a58a259d0d82735 [ 706.154913] env[62736]: DEBUG nova.policy [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 706.349773] env[62736]: INFO nova.scheduler.client.report [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Deleted allocations for instance 3e82fa5d-6610-4d3d-972f-4120aca5e8a7 [ 706.356595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Expecting reply to msg 105cc5074c96487fb7746440f152a0a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 706.369050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 105cc5074c96487fb7746440f152a0a7 [ 706.587202] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 706.588969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 43509d6cb85a450f86dcac1acd3ba86d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 706.624859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43509d6cb85a450f86dcac1acd3ba86d [ 706.681478] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Successfully created port: 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 706.862034] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8dd224a5-7c1b-4ae1-b3b5-fece2290ac2d tempest-AttachInterfacesV270Test-865365601 tempest-AttachInterfacesV270Test-865365601-project-member] Lock "3e82fa5d-6610-4d3d-972f-4120aca5e8a7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 123.186s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 706.862666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 20e161d1796646d8b7f64b499d22fa96 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 706.880766] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20e161d1796646d8b7f64b499d22fa96 [ 706.947171] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5c7e619-9914-4f0a-b9dd-479d71a14380 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.955104] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12ea1b91-1b87-44ee-9fa3-65c494fab865 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.988039] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6c13226-d596-4a7b-bca5-90b24e0788e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 706.992963] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61522c0e-9b76-4d8f-8e00-bea828994493 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.005597] env[62736]: DEBUG nova.compute.provider_tree [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 707.006088] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 26ea278f4504495ba7f05ec3286f2c3e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 707.013457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26ea278f4504495ba7f05ec3286f2c3e [ 707.097291] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a1fd87a273434f1f9597ea39222aaf14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 707.136016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1fd87a273434f1f9597ea39222aaf14 [ 707.364979] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 707.366641] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 8eabd9519de34612adbe0b828622eba6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 707.402438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8eabd9519de34612adbe0b828622eba6 [ 707.509258] env[62736]: DEBUG nova.scheduler.client.report [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 707.511751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 80f298c813554d75998db9a1a9206974 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 707.525087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f298c813554d75998db9a1a9206974 [ 707.600291] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 707.625713] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 707.625967] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 707.626123] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 707.626312] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 707.626452] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 707.626600] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 707.626798] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 707.626950] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 707.627108] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 707.627293] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 707.627493] env[62736]: DEBUG nova.virt.hardware [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 707.628365] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20743034-63df-4628-8a4b-870ce784805e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.635893] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b20e06eb-670f-4001-ba5b-423d62f322b5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 707.890435] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 708.014412] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.446s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 708.014953] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 708.016777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 20ff70aa2cdd4e6cb6fb4d4e1a8283f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.018421] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.221s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 708.019890] env[62736]: INFO nova.compute.claims [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 708.021545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 9b72395f2f7b4f678057b111d0710e75 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.052395] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20ff70aa2cdd4e6cb6fb4d4e1a8283f1 [ 708.061382] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b72395f2f7b4f678057b111d0710e75 [ 708.340459] env[62736]: ERROR nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 708.340459] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 708.340459] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 708.340459] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 708.340459] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 708.340459] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 708.340459] env[62736]: ERROR nova.compute.manager raise self.value [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 708.340459] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 708.340459] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 708.340459] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 708.341180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 708.341180] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 708.341180] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 708.341180] env[62736]: ERROR nova.compute.manager [ 708.341180] env[62736]: Traceback (most recent call last): [ 708.341180] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 708.341180] env[62736]: listener.cb(fileno) [ 708.341180] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 708.341180] env[62736]: result = function(*args, **kwargs) [ 708.341180] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 708.341180] env[62736]: return func(*args, **kwargs) [ 708.341180] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 708.341180] env[62736]: raise e [ 708.341180] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 708.341180] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 708.341180] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 708.341180] env[62736]: created_port_ids = self._update_ports_for_instance( [ 708.341180] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 708.341180] env[62736]: with excutils.save_and_reraise_exception(): [ 708.341180] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 708.341180] env[62736]: self.force_reraise() [ 708.341180] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 708.341180] env[62736]: raise self.value [ 708.341180] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 708.341180] env[62736]: updated_port = self._update_port( [ 708.341180] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 708.341180] env[62736]: _ensure_no_port_binding_failure(port) [ 708.341180] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 708.341180] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 708.342312] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 708.342312] env[62736]: Removing descriptor: 17 [ 708.342312] env[62736]: ERROR nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Traceback (most recent call last): [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] yield resources [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.driver.spawn(context, instance, image_meta, [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self._vmops.spawn(context, instance, image_meta, injected_files, [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 708.342312] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] vm_ref = self.build_virtual_machine(instance, [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] vif_infos = vmwarevif.get_vif_info(self._session, [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] for vif in network_info: [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self._sync_wrapper(fn, *args, **kwargs) [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.wait() [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self[:] = self._gt.wait() [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self._exit_event.wait() [ 708.342876] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] result = hub.switch() [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self.greenlet.switch() [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] result = function(*args, **kwargs) [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return func(*args, **kwargs) [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise e [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] nwinfo = self.network_api.allocate_for_instance( [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 708.343393] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] created_port_ids = self._update_ports_for_instance( [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] with excutils.save_and_reraise_exception(): [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.force_reraise() [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise self.value [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] updated_port = self._update_port( [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] _ensure_no_port_binding_failure(port) [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 708.343920] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise exception.PortBindingFailed(port_id=port['id']) [ 708.344584] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 708.344584] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] [ 708.344584] env[62736]: INFO nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Terminating instance [ 708.344584] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.344584] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 708.344584] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 708.344584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 86ffabd21ad44c0a80c543ff22fb90b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.351894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86ffabd21ad44c0a80c543ff22fb90b8 [ 708.356631] env[62736]: DEBUG nova.compute.manager [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Received event network-changed-0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 708.356819] env[62736]: DEBUG nova.compute.manager [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Refreshing instance network info cache due to event network-changed-0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 708.356995] env[62736]: DEBUG oslo_concurrency.lockutils [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] Acquiring lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 708.520160] env[62736]: DEBUG nova.compute.utils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 708.520841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg e219b2c5bd52493489f3e0ccc790b401 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.521793] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 708.521958] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 708.529402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 321248f7a0814c2083f8c7dc9fd7adbc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.534542] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e219b2c5bd52493489f3e0ccc790b401 [ 708.537072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 321248f7a0814c2083f8c7dc9fd7adbc [ 708.627396] env[62736]: DEBUG nova.policy [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '71886e19fbc749d989c55e17c178862a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7f30e3b24f414007b0066cab5346a3bf', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 708.870169] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 708.946623] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.947148] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9331de392863469497a48f2febe2f75e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 708.962094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9331de392863469497a48f2febe2f75e [ 709.025738] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 709.027933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 306731782c8e4b74a57e5b5f5e7b679c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 709.079341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 306731782c8e4b74a57e5b5f5e7b679c [ 709.267683] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Successfully created port: 135528e4-b55f-4aeb-a74c-3b85b2005c68 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 709.455461] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 709.455977] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 709.456243] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 709.456581] env[62736]: DEBUG oslo_concurrency.lockutils [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] Acquired lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 709.456765] env[62736]: DEBUG nova.network.neutron [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Refreshing network info cache for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 709.457190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] Expecting reply to msg 29a4b97ba7a34976b1fc926189e1a712 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 709.463924] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-af3353bc-6121-4134-9275-54513520a7a7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.465092] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29a4b97ba7a34976b1fc926189e1a712 [ 709.471298] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e591756-e96b-4433-9d86-5f137f752d83 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.496908] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 71ce0048-90fb-460c-a099-7eff5c092d35 could not be found. [ 709.497135] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 709.497311] env[62736]: INFO nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Took 0.04 seconds to destroy the instance on the hypervisor. [ 709.497664] env[62736]: DEBUG oslo.service.loopingcall [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 709.499062] env[62736]: DEBUG nova.compute.manager [-] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 709.499163] env[62736]: DEBUG nova.network.neutron [-] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 709.501633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19e828f0-24a4-443b-b166-416cf0794c11 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.509002] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73a936e2-cfad-4bda-a806-f4ff2f500cc2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.543410] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 9dae5da48a864f1db20617578b49168c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 709.552132] env[62736]: DEBUG nova.network.neutron [-] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 709.552132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6709d996c2314a08b6a29bce3a91adf7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 709.552132] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0989d71e-969f-4c05-b70d-f618bad97380 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.556711] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91ee5f4f-d5e8-41eb-b4c1-e082c788f1ff {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 709.561960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6709d996c2314a08b6a29bce3a91adf7 [ 709.575978] env[62736]: DEBUG nova.compute.provider_tree [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 709.576277] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 5d78dc28029247889816eaa80930cc2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 709.589751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d78dc28029247889816eaa80930cc2d [ 709.599161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9dae5da48a864f1db20617578b49168c [ 709.976727] env[62736]: DEBUG nova.network.neutron [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 710.049517] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 710.053141] env[62736]: DEBUG nova.network.neutron [-] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.053560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1400ddfc6028450db736093cfd4d0694 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.062503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1400ddfc6028450db736093cfd4d0694 [ 710.064531] env[62736]: DEBUG nova.network.neutron [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 710.065011] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] Expecting reply to msg 95c7fc4f87f241398192bbebc998f7f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.085028] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:31:59Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='2c7c48c6-fc2b-4763-813d-b0414ed5bb47',id=36,is_public=True,memory_mb=192,name='tempest-test_resize_flavor_-1405315254',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 710.085028] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 710.085028] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 710.085265] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 710.085265] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 710.085265] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 710.085265] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 710.085265] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 710.085430] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 710.085430] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 710.085430] env[62736]: DEBUG nova.virt.hardware [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 710.085430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95c7fc4f87f241398192bbebc998f7f3 [ 710.085430] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c180282-9192-4c2a-b85a-d7aaa6827643 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.085584] env[62736]: DEBUG nova.scheduler.client.report [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 710.085584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 465ea1de084248f5a730b11d8ca0ad4b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.090286] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c01e545-c957-4d22-87be-032abb703a42 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 710.096884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 465ea1de084248f5a730b11d8ca0ad4b [ 710.324545] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "451b9e70-f202-4a10-8b19-34b6167ef14d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.325386] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "451b9e70-f202-4a10-8b19-34b6167ef14d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.385142] env[62736]: DEBUG nova.compute.manager [req-54378f24-ef23-4ad7-add2-2090a88c9b23 req-1fd6c162-4d06-475d-b6a5-911695a68f6b service nova] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Received event network-vif-deleted-0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 710.555812] env[62736]: INFO nova.compute.manager [-] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Took 1.06 seconds to deallocate network for instance. [ 710.558353] env[62736]: DEBUG nova.compute.claims [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 710.558539] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.567170] env[62736]: DEBUG oslo_concurrency.lockutils [req-7cb7effa-2173-4ed0-87ca-dc9e276ef80f req-205d615e-187b-49a1-8370-c7e1952f99b9 service nova] Releasing lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 710.586226] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.568s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.586809] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 710.588682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg c4f942954c9e4a4cb22a8886ce2949db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.589823] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 16.436s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.590035] env[62736]: DEBUG nova.objects.instance [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lazy-loading 'resources' on Instance uuid b42fe7a4-ec30-441f-b5dc-1f705ee5c881 {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 710.590340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 95a87e36d3d14af68e454e5408cf8ccd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.597448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95a87e36d3d14af68e454e5408cf8ccd [ 710.626877] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4f942954c9e4a4cb22a8886ce2949db [ 710.634405] env[62736]: ERROR nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 710.634405] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 710.634405] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 710.634405] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 710.634405] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 710.634405] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 710.634405] env[62736]: ERROR nova.compute.manager raise self.value [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 710.634405] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 710.634405] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 710.634405] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 710.634865] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 710.634865] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 710.634865] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 710.634865] env[62736]: ERROR nova.compute.manager [ 710.634865] env[62736]: Traceback (most recent call last): [ 710.634865] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 710.634865] env[62736]: listener.cb(fileno) [ 710.634865] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 710.634865] env[62736]: result = function(*args, **kwargs) [ 710.634865] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 710.634865] env[62736]: return func(*args, **kwargs) [ 710.634865] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 710.634865] env[62736]: raise e [ 710.634865] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 710.634865] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 710.634865] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 710.634865] env[62736]: created_port_ids = self._update_ports_for_instance( [ 710.634865] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 710.634865] env[62736]: with excutils.save_and_reraise_exception(): [ 710.634865] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 710.634865] env[62736]: self.force_reraise() [ 710.634865] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 710.634865] env[62736]: raise self.value [ 710.634865] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 710.634865] env[62736]: updated_port = self._update_port( [ 710.634865] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 710.634865] env[62736]: _ensure_no_port_binding_failure(port) [ 710.634865] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 710.634865] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 710.635605] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 710.635605] env[62736]: Removing descriptor: 17 [ 710.635605] env[62736]: ERROR nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Traceback (most recent call last): [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] yield resources [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.driver.spawn(context, instance, image_meta, [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 710.635605] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] vm_ref = self.build_virtual_machine(instance, [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] vif_infos = vmwarevif.get_vif_info(self._session, [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] for vif in network_info: [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self._sync_wrapper(fn, *args, **kwargs) [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.wait() [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self[:] = self._gt.wait() [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self._exit_event.wait() [ 710.635910] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] result = hub.switch() [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self.greenlet.switch() [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] result = function(*args, **kwargs) [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return func(*args, **kwargs) [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise e [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] nwinfo = self.network_api.allocate_for_instance( [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 710.636255] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] created_port_ids = self._update_ports_for_instance( [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] with excutils.save_and_reraise_exception(): [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.force_reraise() [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise self.value [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] updated_port = self._update_port( [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] _ensure_no_port_binding_failure(port) [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 710.636578] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise exception.PortBindingFailed(port_id=port['id']) [ 710.636885] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 710.636885] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] [ 710.636885] env[62736]: INFO nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Terminating instance [ 710.637726] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 710.637878] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquired lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 710.638038] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 710.638454] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 389a22fe02e24533b83cafede0992c29 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 710.644973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 389a22fe02e24533b83cafede0992c29 [ 711.095622] env[62736]: DEBUG nova.compute.utils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 711.096272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg d10275048120429789c47471a8103362 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 711.097599] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 711.097934] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 711.108731] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d10275048120429789c47471a8103362 [ 711.161284] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 711.196093] env[62736]: DEBUG nova.policy [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '520a746e985543d887fe9436704586f2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3eb283a2654947978d60cee9a47d568f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 711.216044] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 711.216842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg d5dcc115a721454baf433e38e7b8d0c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 711.225806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5dcc115a721454baf433e38e7b8d0c3 [ 711.493033] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a283295-e245-4653-abeb-247b9862aad2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.500591] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e05f03cb-b914-4aae-a8f4-3e49b3b1bddf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.531644] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06e3584d-f5f6-4e5b-ade8-70242fe787fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.538640] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60400a3e-e07d-433f-a769-7865c69faaa9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.551737] env[62736]: DEBUG nova.compute.provider_tree [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 711.552262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 2c8ec1cb8f2146d3ae56a58544c13930 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 711.559617] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Successfully created port: 749951e6-e808-4d96-b4e6-bbcce0326a32 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 711.561611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c8ec1cb8f2146d3ae56a58544c13930 [ 711.600590] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 711.602376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg a25d3967f7ae4078baa2ca11abb27698 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 711.636630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a25d3967f7ae4078baa2ca11abb27698 [ 711.719027] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Releasing lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 711.719521] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 711.719713] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 711.720023] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3f3a94eb-e177-48e8-ba88-66d5db3040b7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.728630] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f92753b-d3f6-4c46-8e05-8b9fb2dabce2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 711.749967] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 20d9b891-9c2a-479c-8fd8-93405480dee4 could not be found. [ 711.750198] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 711.750443] env[62736]: INFO nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Took 0.03 seconds to destroy the instance on the hypervisor. [ 711.750700] env[62736]: DEBUG oslo.service.loopingcall [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 711.750906] env[62736]: DEBUG nova.compute.manager [-] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 711.751000] env[62736]: DEBUG nova.network.neutron [-] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 711.769667] env[62736]: DEBUG nova.network.neutron [-] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 711.770169] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c33d3f2e1f8e433a8e430d18f4add30a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 711.784974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c33d3f2e1f8e433a8e430d18f4add30a [ 712.060252] env[62736]: DEBUG nova.scheduler.client.report [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 712.063001] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg d4451520830444df88b9f2938c47d5d8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.079337] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4451520830444df88b9f2938c47d5d8 [ 712.106940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg d511297c572e4ef985f8e17429f7f9c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.138185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d511297c572e4ef985f8e17429f7f9c9 [ 712.284860] env[62736]: DEBUG nova.network.neutron [-] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 712.285333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 26f2e137735a4f1791db26ed92143ae7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.293361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26f2e137735a4f1791db26ed92143ae7 [ 712.419864] env[62736]: DEBUG nova.compute.manager [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Received event network-changed-135528e4-b55f-4aeb-a74c-3b85b2005c68 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 712.420076] env[62736]: DEBUG nova.compute.manager [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Refreshing instance network info cache due to event network-changed-135528e4-b55f-4aeb-a74c-3b85b2005c68. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 712.420318] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Acquiring lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 712.420488] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Acquired lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 712.420661] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Refreshing network info cache for port 135528e4-b55f-4aeb-a74c-3b85b2005c68 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 712.421069] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Expecting reply to msg 65af39b8e27b46b2837d774cbbdc769d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.428625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65af39b8e27b46b2837d774cbbdc769d [ 712.565503] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.976s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 712.567865] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.636s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 712.569854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 4b5893f710674886968de2131992ca8c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.588440] env[62736]: INFO nova.scheduler.client.report [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Deleted allocations for instance b42fe7a4-ec30-441f-b5dc-1f705ee5c881 [ 712.591051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg 10f4465ebd6445fe9a1c9af10d061386 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.617384] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b5893f710674886968de2131992ca8c [ 712.623586] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 712.649518] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 712.649777] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 712.649991] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 712.650099] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 712.650242] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 712.650385] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 712.650624] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 712.650780] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 712.650952] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 712.651091] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 712.651258] env[62736]: DEBUG nova.virt.hardware [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 712.652453] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e983d4c-6100-4864-b41b-83de3104c391 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.661423] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f0987ff-f341-4ad8-b6bc-dc9a0a573c74 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 712.683767] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10f4465ebd6445fe9a1c9af10d061386 [ 712.787868] env[62736]: INFO nova.compute.manager [-] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Took 1.04 seconds to deallocate network for instance. [ 712.790250] env[62736]: DEBUG nova.compute.claims [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 712.790408] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 712.810868] env[62736]: ERROR nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 712.810868] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 712.810868] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 712.810868] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 712.810868] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 712.810868] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 712.810868] env[62736]: ERROR nova.compute.manager raise self.value [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 712.810868] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 712.810868] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 712.810868] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 712.811390] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 712.811390] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 712.811390] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 712.811390] env[62736]: ERROR nova.compute.manager [ 712.811390] env[62736]: Traceback (most recent call last): [ 712.811390] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 712.811390] env[62736]: listener.cb(fileno) [ 712.811390] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 712.811390] env[62736]: result = function(*args, **kwargs) [ 712.811390] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 712.811390] env[62736]: return func(*args, **kwargs) [ 712.811390] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 712.811390] env[62736]: raise e [ 712.811390] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 712.811390] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 712.811390] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 712.811390] env[62736]: created_port_ids = self._update_ports_for_instance( [ 712.811390] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 712.811390] env[62736]: with excutils.save_and_reraise_exception(): [ 712.811390] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 712.811390] env[62736]: self.force_reraise() [ 712.811390] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 712.811390] env[62736]: raise self.value [ 712.811390] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 712.811390] env[62736]: updated_port = self._update_port( [ 712.811390] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 712.811390] env[62736]: _ensure_no_port_binding_failure(port) [ 712.811390] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 712.811390] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 712.812249] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 712.812249] env[62736]: Removing descriptor: 17 [ 712.812249] env[62736]: ERROR nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Traceback (most recent call last): [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] yield resources [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.driver.spawn(context, instance, image_meta, [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 712.812249] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] vm_ref = self.build_virtual_machine(instance, [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] vif_infos = vmwarevif.get_vif_info(self._session, [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] for vif in network_info: [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self._sync_wrapper(fn, *args, **kwargs) [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.wait() [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self[:] = self._gt.wait() [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self._exit_event.wait() [ 712.812726] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] result = hub.switch() [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self.greenlet.switch() [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] result = function(*args, **kwargs) [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return func(*args, **kwargs) [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise e [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] nwinfo = self.network_api.allocate_for_instance( [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 712.813114] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] created_port_ids = self._update_ports_for_instance( [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] with excutils.save_and_reraise_exception(): [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.force_reraise() [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise self.value [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] updated_port = self._update_port( [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] _ensure_no_port_binding_failure(port) [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 712.813538] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise exception.PortBindingFailed(port_id=port['id']) [ 712.813944] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 712.813944] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] [ 712.813944] env[62736]: INFO nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Terminating instance [ 712.813944] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquiring lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 712.814068] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquired lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 712.814222] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 712.814671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 28b66275267147a7a16dc56153cfffc7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 712.821613] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28b66275267147a7a16dc56153cfffc7 [ 712.936961] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 713.012058] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 713.012612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Expecting reply to msg 5fa668a538644383a715c19786ce7ff5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.026584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fa668a538644383a715c19786ce7ff5 [ 713.099567] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Lock "b42fe7a4-ec30-441f-b5dc-1f705ee5c881" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 22.657s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.100128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d2733f76-1972-4234-885e-d98f4fb19f24 tempest-ServerShowV247Test-815835844 tempest-ServerShowV247Test-815835844-project-member] Expecting reply to msg edabf2e57e20407c9ffa9499897a2100 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.125109] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edabf2e57e20407c9ffa9499897a2100 [ 713.340246] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 713.428042] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 713.428550] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 8ac6f0e0422f42ad8d20ac0ec18b285e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.435904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ac6f0e0422f42ad8d20ac0ec18b285e [ 713.469716] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-280cf147-b503-44f2-b0f0-f0e39d85907f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.477433] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71c43e04-ccd5-42cc-851c-975b60c689af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.507963] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f2485ab-d51b-4259-a386-bf346783dbdc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.515125] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b4ebea1-78f9-4a8a-8696-db924b784fec {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.520298] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Releasing lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 713.520690] env[62736]: DEBUG nova.compute.manager [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Received event network-vif-deleted-135528e4-b55f-4aeb-a74c-3b85b2005c68 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 713.520924] env[62736]: DEBUG nova.compute.manager [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Received event network-changed-749951e6-e808-4d96-b4e6-bbcce0326a32 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 713.521082] env[62736]: DEBUG nova.compute.manager [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Refreshing instance network info cache due to event network-changed-749951e6-e808-4d96-b4e6-bbcce0326a32. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 713.521252] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Acquiring lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 713.532195] env[62736]: DEBUG nova.compute.provider_tree [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 713.532669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 3e47339f911a47c1975bb7dd578f8468 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.539554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e47339f911a47c1975bb7dd578f8468 [ 713.930620] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Releasing lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 713.931267] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 713.931524] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 713.931871] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Acquired lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 713.932064] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Refreshing network info cache for port 749951e6-e808-4d96-b4e6-bbcce0326a32 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 713.932497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Expecting reply to msg 9f17ce9b481e4646b68bc98bab670d3f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.933321] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b2aa3600-184e-4cdd-ac7b-71778f591c3e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.940215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f17ce9b481e4646b68bc98bab670d3f [ 713.944032] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef3a82b6-5067-4a1e-bfc7-d2c093c7c9b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.965629] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 48a0d4f8-4fab-454b-b065-e552647fba3f could not be found. [ 713.965855] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 713.966030] env[62736]: INFO nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Took 0.03 seconds to destroy the instance on the hypervisor. [ 713.966266] env[62736]: DEBUG oslo.service.loopingcall [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 713.966501] env[62736]: DEBUG nova.compute.manager [-] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 713.966598] env[62736]: DEBUG nova.network.neutron [-] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 713.986094] env[62736]: DEBUG nova.network.neutron [-] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 713.986574] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c43232121b4045439e6a7dc94db60685 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 713.994393] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c43232121b4045439e6a7dc94db60685 [ 714.035248] env[62736]: DEBUG nova.scheduler.client.report [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 714.038070] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 4aa2b4f64fa44fe0ab72a5d193439c1a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 714.053263] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4aa2b4f64fa44fe0ab72a5d193439c1a [ 714.448478] env[62736]: DEBUG nova.compute.manager [req-eb64bcd0-c20e-4edd-aa56-d865a24e8277 req-fb5985d0-5a99-4eca-9b67-afe6560e41fe service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Received event network-vif-deleted-749951e6-e808-4d96-b4e6-bbcce0326a32 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 714.462525] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 714.474079] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 714.474426] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 714.489252] env[62736]: DEBUG nova.network.neutron [-] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 714.489252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 709837e72954468883bbaea6d45cbcb7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 714.499016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 709837e72954468883bbaea6d45cbcb7 [ 714.543058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.975s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 714.543773] env[62736]: ERROR nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] Traceback (most recent call last): [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.driver.spawn(context, instance, image_meta, [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] vm_ref = self.build_virtual_machine(instance, [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] vif_infos = vmwarevif.get_vif_info(self._session, [ 714.543773] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] for vif in network_info: [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return self._sync_wrapper(fn, *args, **kwargs) [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.wait() [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self[:] = self._gt.wait() [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return self._exit_event.wait() [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] current.throw(*self._exc) [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 714.544200] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] result = function(*args, **kwargs) [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] return func(*args, **kwargs) [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise e [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] nwinfo = self.network_api.allocate_for_instance( [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] created_port_ids = self._update_ports_for_instance( [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] with excutils.save_and_reraise_exception(): [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] self.force_reraise() [ 714.544587] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise self.value [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] updated_port = self._update_port( [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] _ensure_no_port_binding_failure(port) [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] raise exception.PortBindingFailed(port_id=port['id']) [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] nova.exception.PortBindingFailed: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. [ 714.544963] env[62736]: ERROR nova.compute.manager [instance: 15215512-da0c-407f-a352-43c051698f2d] [ 714.544963] env[62736]: DEBUG nova.compute.utils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 714.546131] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.225s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 714.547852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 41b289a8441d4cc69ce94ee3aa69d3ac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 714.549027] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Build of instance 15215512-da0c-407f-a352-43c051698f2d was re-scheduled: Binding failed for port 56565dec-4584-4838-afdb-14c02b5da697, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 714.549527] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 714.549757] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquiring lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 714.549901] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Acquired lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 714.550055] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 714.550450] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 38663b91090847ba93ae68af154675b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 714.569850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38663b91090847ba93ae68af154675b0 [ 714.570976] env[62736]: DEBUG nova.network.neutron [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 714.572128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Expecting reply to msg 1792f9e2e4e8436d833ee1ac81fcf1ba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 714.583370] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1792f9e2e4e8436d833ee1ac81fcf1ba [ 714.587185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41b289a8441d4cc69ce94ee3aa69d3ac [ 714.992562] env[62736]: INFO nova.compute.manager [-] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Took 1.03 seconds to deallocate network for instance. [ 714.994906] env[62736]: DEBUG nova.compute.claims [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 714.995081] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 715.073997] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 715.075255] env[62736]: DEBUG oslo_concurrency.lockutils [req-1cd5efab-225c-4f6b-a7b5-d66d6de37476 req-e351e743-a819-4e49-8d06-39380fe26f6e service nova] Releasing lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 715.189998] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.190543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg d92f9232e7ac4b08a344b85a6b9e5463 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 715.201505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d92f9232e7ac4b08a344b85a6b9e5463 [ 715.430374] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb5fddff-17b5-4fa3-8339-cd4e1a53cc9f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.437869] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-978d55cf-5593-4610-a788-89891185ddf1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.467485] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-413627cd-2b61-4e92-a929-04aaed28864f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.474198] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cfa72ff-70f1-4d51-803a-6e3d70cfb950 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 715.487258] env[62736]: DEBUG nova.compute.provider_tree [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 715.487770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg ba584e5bc1504163ba2cdfeaadf4cfd0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 715.494859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ba584e5bc1504163ba2cdfeaadf4cfd0 [ 715.692872] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Releasing lock "refresh_cache-15215512-da0c-407f-a352-43c051698f2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 715.693136] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 715.693322] env[62736]: DEBUG nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 715.693487] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 715.707208] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 715.707771] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg e30cd15cc7cd4aba8c85193dfc026a87 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 715.716608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e30cd15cc7cd4aba8c85193dfc026a87 [ 715.992025] env[62736]: DEBUG nova.scheduler.client.report [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 715.992815] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg f1fda8ab777d4461b575d723963a1a86 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 716.006350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1fda8ab777d4461b575d723963a1a86 [ 716.210238] env[62736]: DEBUG nova.network.neutron [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 716.210815] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 9f36efca48724e509daa4697101d4f4a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 716.218706] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f36efca48724e509daa4697101d4f4a [ 716.495357] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.950s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 716.496156] env[62736]: ERROR nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Traceback (most recent call last): [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.driver.spawn(context, instance, image_meta, [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] vm_ref = self.build_virtual_machine(instance, [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] vif_infos = vmwarevif.get_vif_info(self._session, [ 716.496156] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] for vif in network_info: [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return self._sync_wrapper(fn, *args, **kwargs) [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.wait() [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self[:] = self._gt.wait() [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return self._exit_event.wait() [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] current.throw(*self._exc) [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 716.496527] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] result = function(*args, **kwargs) [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] return func(*args, **kwargs) [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise e [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] nwinfo = self.network_api.allocate_for_instance( [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] created_port_ids = self._update_ports_for_instance( [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] with excutils.save_and_reraise_exception(): [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] self.force_reraise() [ 716.496926] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise self.value [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] updated_port = self._update_port( [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] _ensure_no_port_binding_failure(port) [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] raise exception.PortBindingFailed(port_id=port['id']) [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] nova.exception.PortBindingFailed: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. [ 716.497494] env[62736]: ERROR nova.compute.manager [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] [ 716.497494] env[62736]: DEBUG nova.compute.utils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 716.498119] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.681s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 716.499934] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 273c0dfe3b8c4eebad9c86036f77d4aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 716.501168] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Build of instance b88d76a9-6a30-4d5f-a5cd-f698e005dab5 was re-scheduled: Binding failed for port ed90fdc0-76f6-4c25-82fe-d55ce7fa35e2, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 716.501701] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 716.501939] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquiring lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 716.502091] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Acquired lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 716.502247] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 716.502615] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 3644b4a359114e4186483e8efc51ce9d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 716.518231] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3644b4a359114e4186483e8efc51ce9d [ 716.535463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 273c0dfe3b8c4eebad9c86036f77d4aa [ 716.714596] env[62736]: INFO nova.compute.manager [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] [instance: 15215512-da0c-407f-a352-43c051698f2d] Took 1.02 seconds to deallocate network for instance. [ 716.715864] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 49eb54fd8b0c44a2b1e2e2569c5ec0bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 716.747290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49eb54fd8b0c44a2b1e2e2569c5ec0bd [ 717.031131] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 717.105953] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 717.106467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg b257bf9ef24d47aeb1a8e676c7f1fd0b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.115643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b257bf9ef24d47aeb1a8e676c7f1fd0b [ 717.220983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg d9395b475f9f4f128cd6bfd296d21fc7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.257757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9395b475f9f4f128cd6bfd296d21fc7 [ 717.418461] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90ceaa0b-18ed-4916-a9ae-26f68f0dc4b6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.426569] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf4b85ad-99bc-4033-89b6-cd675a7a21ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.457659] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcccbdad-1671-44ec-8ed3-5d0bc573dad5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.465954] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8a0dc6c-4c1f-4828-b4e3-228d68723378 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 717.478125] env[62736]: DEBUG nova.compute.provider_tree [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 717.478802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 72d44df72713415e80048b30949309e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.485875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72d44df72713415e80048b30949309e3 [ 717.608839] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Releasing lock "refresh_cache-b88d76a9-6a30-4d5f-a5cd-f698e005dab5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 717.609526] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 717.609854] env[62736]: DEBUG nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 717.610169] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 717.623435] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 717.624344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 50e31e0c1c014a219f35f6d5cbef0e5e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.631418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50e31e0c1c014a219f35f6d5cbef0e5e [ 717.746064] env[62736]: INFO nova.scheduler.client.report [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Deleted allocations for instance 15215512-da0c-407f-a352-43c051698f2d [ 717.750994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Expecting reply to msg 3e9cc43b602147d6a838d7e3503a4a55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.764525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e9cc43b602147d6a838d7e3503a4a55 [ 717.981490] env[62736]: DEBUG nova.scheduler.client.report [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 717.983978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg bfa66b4ceb9248bf879572bb4f76903c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 717.994878] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfa66b4ceb9248bf879572bb4f76903c [ 718.127141] env[62736]: DEBUG nova.network.neutron [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 718.127674] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 1071462b1a3b46b9b15021cb8dc201c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.137286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1071462b1a3b46b9b15021cb8dc201c9 [ 718.253163] env[62736]: DEBUG oslo_concurrency.lockutils [None req-492d4bbb-c067-4c11-aff5-c1d7608618d0 tempest-ServersTestFqdnHostnames-893793326 tempest-ServersTestFqdnHostnames-893793326-project-member] Lock "15215512-da0c-407f-a352-43c051698f2d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 133.490s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.253773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 2591557fecc84c6497fe7aa8bc2784c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.264669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2591557fecc84c6497fe7aa8bc2784c5 [ 718.487278] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.989s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.489882] env[62736]: ERROR nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Traceback (most recent call last): [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.driver.spawn(context, instance, image_meta, [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] vm_ref = self.build_virtual_machine(instance, [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] vif_infos = vmwarevif.get_vif_info(self._session, [ 718.489882] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] for vif in network_info: [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return self._sync_wrapper(fn, *args, **kwargs) [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.wait() [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self[:] = self._gt.wait() [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return self._exit_event.wait() [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] current.throw(*self._exc) [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 718.490510] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] result = function(*args, **kwargs) [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] return func(*args, **kwargs) [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise e [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] nwinfo = self.network_api.allocate_for_instance( [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] created_port_ids = self._update_ports_for_instance( [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] with excutils.save_and_reraise_exception(): [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] self.force_reraise() [ 718.491088] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise self.value [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] updated_port = self._update_port( [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] _ensure_no_port_binding_failure(port) [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] raise exception.PortBindingFailed(port_id=port['id']) [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] nova.exception.PortBindingFailed: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. [ 718.491697] env[62736]: ERROR nova.compute.manager [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] [ 718.491697] env[62736]: DEBUG nova.compute.utils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 718.492179] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.245s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.492179] env[62736]: INFO nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 718.494308] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg e9f03288b9fc40968f60f226e4d00c5e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.495668] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Build of instance eeb7cdc7-427f-496f-be5d-60151a4bb0f3 was re-scheduled: Binding failed for port a20f45f8-34a8-433b-b086-d64d4a77b648, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 718.496177] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 718.496413] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 718.496560] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquired lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 718.496713] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 718.497087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg a08d173de9444f5797242e5030b04e0d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.508595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a08d173de9444f5797242e5030b04e0d [ 718.527741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9f03288b9fc40968f60f226e4d00c5e [ 718.630907] env[62736]: INFO nova.compute.manager [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] [instance: b88d76a9-6a30-4d5f-a5cd-f698e005dab5] Took 1.02 seconds to deallocate network for instance. [ 718.632704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 88a4ce227cdd4d229ffc3c10cc56ade2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.663626] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88a4ce227cdd4d229ffc3c10cc56ade2 [ 718.756055] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 718.757767] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 672d4ebfbd4140b5aef0390d65b51ef9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 718.815571] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 672d4ebfbd4140b5aef0390d65b51ef9 [ 718.999846] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg fd33cab3d0f74dd998eab8bd70a1710a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.011497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd33cab3d0f74dd998eab8bd70a1710a [ 719.019661] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 719.137557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 3259ec4917af42c6baf42d7828221c72 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.162775] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 719.163271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg f16c1cc15a2d40d29ed9c3b8f996b4bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.171556] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f16c1cc15a2d40d29ed9c3b8f996b4bb [ 719.175422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3259ec4917af42c6baf42d7828221c72 [ 719.278033] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 719.658334] env[62736]: INFO nova.scheduler.client.report [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Deleted allocations for instance b88d76a9-6a30-4d5f-a5cd-f698e005dab5 [ 719.669024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Expecting reply to msg 0b3228cc21834dc4b77d21b45ee6dbf5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.669024] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Releasing lock "refresh_cache-eeb7cdc7-427f-496f-be5d-60151a4bb0f3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 719.669024] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 719.669024] env[62736]: DEBUG nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 719.669024] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 719.677763] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b3228cc21834dc4b77d21b45ee6dbf5 [ 719.680890] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 719.681432] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 1387f149a0a84736809f8a8219c8cc86 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.699790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1387f149a0a84736809f8a8219c8cc86 [ 719.895382] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e00d3fc-4d9c-416a-a3e8-7d2c1e7b2b31 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.903268] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc841781-d7da-457a-bac9-d8efa700c09c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.933431] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-276e4d6a-202a-40f9-9d5c-3016f55cbfe8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.940517] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e1b59d9c-1703-47da-9f71-3578af41f72e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 719.952807] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 719.953295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 938d27b453cc481490fa14cf06dd8862 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 719.960100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 938d27b453cc481490fa14cf06dd8862 [ 720.167107] env[62736]: DEBUG oslo_concurrency.lockutils [None req-669cbdde-a42e-4038-873d-964c62b5e74a tempest-ServersAdminNegativeTestJSON-1339964137 tempest-ServersAdminNegativeTestJSON-1339964137-project-member] Lock "b88d76a9-6a30-4d5f-a5cd-f698e005dab5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 134.403s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.167107] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 4d647a09acaf427899f9e8901156cf0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.177814] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d647a09acaf427899f9e8901156cf0a [ 720.184055] env[62736]: DEBUG nova.network.neutron [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 720.184347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg d8539dac2c3a44098058f1641f5ccb61 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.192245] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8539dac2c3a44098058f1641f5ccb61 [ 720.457002] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 720.459262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 4a1bb790c55848c6a320b4fd51c2a3de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.471636] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a1bb790c55848c6a320b4fd51c2a3de [ 720.668975] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 720.670948] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 52ea552fcc4a4d30a03b35f580b909fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.686959] env[62736]: INFO nova.compute.manager [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: eeb7cdc7-427f-496f-be5d-60151a4bb0f3] Took 1.02 seconds to deallocate network for instance. [ 720.688885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 1dec00bb12ba421588430952d2c9a679 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.707357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52ea552fcc4a4d30a03b35f580b909fa [ 720.731989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1dec00bb12ba421588430952d2c9a679 [ 720.961686] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.472s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.962220] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 720.963811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg cc26fcc854af423193a3c81cf0e2a702 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.965253] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.137s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.966971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 94af60d598f74723a5208eb054c12e01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 720.997302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc26fcc854af423193a3c81cf0e2a702 [ 720.998966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94af60d598f74723a5208eb054c12e01 [ 721.194656] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg f1fb8732c5764e42a97cd9de2ba1fe97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 721.196626] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.243939] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1fb8732c5764e42a97cd9de2ba1fe97 [ 721.469830] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 721.470505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 14abd4d46d324d368ffa512a18f73aa9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 721.476616] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 721.476669] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 721.482084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14abd4d46d324d368ffa512a18f73aa9 [ 721.551069] env[62736]: DEBUG nova.policy [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c2b0969731654200b597f9d6398c2cfb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '78c69a6dc9124f71b9a1f84fb2da5d4d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 721.724422] env[62736]: INFO nova.scheduler.client.report [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Deleted allocations for instance eeb7cdc7-427f-496f-be5d-60151a4bb0f3 [ 721.735823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 27c6cae739444be8bef810f65ee30dee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 721.760629] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27c6cae739444be8bef810f65ee30dee [ 721.906638] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7aee69c6-cb9b-4447-b5ef-68cc341e0b24 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.914395] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e71a4c0-4d16-44db-8431-06bb5c9031a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.946405] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-107fa3ac-9867-486e-ac9b-7f28b3e7c00d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.953714] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7fbad3f-60eb-4dfc-81b9-d6683d80dbeb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 721.966645] env[62736]: DEBUG nova.compute.provider_tree [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 721.967157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg a33e22ddeb1046d6b3f0fa2f81ca82bc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 721.974523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a33e22ddeb1046d6b3f0fa2f81ca82bc [ 721.975157] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 721.976823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 659829edf1724acaa5b8843ffc0aeab4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.009948] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 659829edf1724acaa5b8843ffc0aeab4 [ 722.169598] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Successfully created port: ebf4f464-5af2-4630-82e7-8b0520106de4 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 722.244382] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25baa20c-fe6e-4086-9fc3-4043d24bed79 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "eeb7cdc7-427f-496f-be5d-60151a4bb0f3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 135.626s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 722.244998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 91ba18318bd349d09f891e4392ef6f73 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.261789] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91ba18318bd349d09f891e4392ef6f73 [ 722.470761] env[62736]: DEBUG nova.scheduler.client.report [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 722.472993] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 40e2af099cda40168f56c4b0d1c2c81d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.480728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 6fcf2430cc3e48f59ce92495e72790ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.497884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40e2af099cda40168f56c4b0d1c2c81d [ 722.527641] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fcf2430cc3e48f59ce92495e72790ea [ 722.747877] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 722.749913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 31eb6ea075c749dd842cb207b264c7df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.804942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31eb6ea075c749dd842cb207b264c7df [ 722.975594] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.011s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 722.976259] env[62736]: ERROR nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Traceback (most recent call last): [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.driver.spawn(context, instance, image_meta, [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] vm_ref = self.build_virtual_machine(instance, [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] vif_infos = vmwarevif.get_vif_info(self._session, [ 722.976259] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] for vif in network_info: [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self._sync_wrapper(fn, *args, **kwargs) [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.wait() [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self[:] = self._gt.wait() [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self._exit_event.wait() [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] result = hub.switch() [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 722.976614] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return self.greenlet.switch() [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] result = function(*args, **kwargs) [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] return func(*args, **kwargs) [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise e [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] nwinfo = self.network_api.allocate_for_instance( [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] created_port_ids = self._update_ports_for_instance( [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] with excutils.save_and_reraise_exception(): [ 722.976963] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] self.force_reraise() [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise self.value [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] updated_port = self._update_port( [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] _ensure_no_port_binding_failure(port) [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] raise exception.PortBindingFailed(port_id=port['id']) [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] nova.exception.PortBindingFailed: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. [ 722.977303] env[62736]: ERROR nova.compute.manager [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] [ 722.977605] env[62736]: DEBUG nova.compute.utils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 722.978273] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.165s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 722.979812] env[62736]: INFO nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 722.981299] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 307b683c3d1c4dcaaff5dab8c9d64b2e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.982441] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Build of instance 1754ee22-ae95-4439-a6c0-200fd153bad0 was re-scheduled: Binding failed for port 9ba330ad-0f59-4b8a-898d-9edc8943371d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 722.982876] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 722.983099] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquiring lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 722.983245] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Acquired lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 722.983399] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 722.983761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 35f8da8a3b0e47fab2aa538d4fb9fa46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 722.987052] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 722.996407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35f8da8a3b0e47fab2aa538d4fb9fa46 [ 723.009526] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 723.010065] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 723.010299] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 723.010827] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 723.011105] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 723.011324] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 723.011589] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 723.014969] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 723.018363] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 723.018621] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 723.021569] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 723.023908] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8669c295-2899-4619-8d39-d44929af446e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.027252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 307b683c3d1c4dcaaff5dab8c9d64b2e [ 723.034835] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7368c1b-7fca-44b9-bd60-6403e39c5a09 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 723.096214] env[62736]: DEBUG nova.compute.manager [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Received event network-changed-ebf4f464-5af2-4630-82e7-8b0520106de4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 723.096457] env[62736]: DEBUG nova.compute.manager [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Refreshing instance network info cache due to event network-changed-ebf4f464-5af2-4630-82e7-8b0520106de4. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 723.096651] env[62736]: DEBUG oslo_concurrency.lockutils [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] Acquiring lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 723.097022] env[62736]: DEBUG oslo_concurrency.lockutils [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] Acquired lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 723.097229] env[62736]: DEBUG nova.network.neutron [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Refreshing network info cache for port ebf4f464-5af2-4630-82e7-8b0520106de4 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 723.097680] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] Expecting reply to msg dea52f47aafe4a908a3652d43d95a59f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 723.107664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dea52f47aafe4a908a3652d43d95a59f [ 723.273050] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.330962] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 723.330962] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 723.330962] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 723.330962] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 723.330962] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 723.330962] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 723.330962] env[62736]: ERROR nova.compute.manager raise self.value [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 723.330962] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 723.330962] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 723.330962] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 723.331449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 723.331449] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 723.331449] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 723.331449] env[62736]: ERROR nova.compute.manager [ 723.331449] env[62736]: Traceback (most recent call last): [ 723.331449] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 723.331449] env[62736]: listener.cb(fileno) [ 723.331449] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 723.331449] env[62736]: result = function(*args, **kwargs) [ 723.331449] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 723.331449] env[62736]: return func(*args, **kwargs) [ 723.331449] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 723.331449] env[62736]: raise e [ 723.331449] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 723.331449] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 723.331449] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 723.331449] env[62736]: created_port_ids = self._update_ports_for_instance( [ 723.331449] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 723.331449] env[62736]: with excutils.save_and_reraise_exception(): [ 723.331449] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 723.331449] env[62736]: self.force_reraise() [ 723.331449] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 723.331449] env[62736]: raise self.value [ 723.331449] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 723.331449] env[62736]: updated_port = self._update_port( [ 723.331449] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 723.331449] env[62736]: _ensure_no_port_binding_failure(port) [ 723.331449] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 723.331449] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 723.332377] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 723.332377] env[62736]: Removing descriptor: 17 [ 723.332377] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Traceback (most recent call last): [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] yield resources [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.driver.spawn(context, instance, image_meta, [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 723.332377] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] vm_ref = self.build_virtual_machine(instance, [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] vif_infos = vmwarevif.get_vif_info(self._session, [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] for vif in network_info: [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self._sync_wrapper(fn, *args, **kwargs) [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.wait() [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self[:] = self._gt.wait() [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self._exit_event.wait() [ 723.333088] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] result = hub.switch() [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self.greenlet.switch() [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] result = function(*args, **kwargs) [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return func(*args, **kwargs) [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise e [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] nwinfo = self.network_api.allocate_for_instance( [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 723.333494] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] created_port_ids = self._update_ports_for_instance( [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] with excutils.save_and_reraise_exception(): [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.force_reraise() [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise self.value [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] updated_port = self._update_port( [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] _ensure_no_port_binding_failure(port) [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 723.333899] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise exception.PortBindingFailed(port_id=port['id']) [ 723.334276] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 723.334276] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] [ 723.334276] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Terminating instance [ 723.335245] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 723.490183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg a93a40a09a37408481ea9b1658a33c94 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 723.497620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a93a40a09a37408481ea9b1658a33c94 [ 723.510671] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.619925] env[62736]: DEBUG nova.network.neutron [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.678640] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.679177] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg d32d364d47b5401fbaec582b35818629 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 723.687535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d32d364d47b5401fbaec582b35818629 [ 723.718798] env[62736]: DEBUG nova.network.neutron [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 723.719379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] Expecting reply to msg 90a521b8cd67475daaac563035e8688c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 723.735187] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90a521b8cd67475daaac563035e8688c [ 724.181582] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Releasing lock "refresh_cache-1754ee22-ae95-4439-a6c0-200fd153bad0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 724.181872] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 724.182187] env[62736]: DEBUG nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 724.182187] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 724.200197] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 724.200783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 5f0c3f918bc942b38b8235999ff00633 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.210812] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f0c3f918bc942b38b8235999ff00633 [ 724.221658] env[62736]: DEBUG oslo_concurrency.lockutils [req-c5078358-415a-4c46-8520-c519fd6ef27a req-c7e278b7-c288-4ba3-bf2c-eaefe15b7d97 service nova] Releasing lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 724.221818] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 724.221999] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 724.222429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 192d014bf2fc4bb1b769a9cd264982b5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.230619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 192d014bf2fc4bb1b769a9cd264982b5 [ 724.372928] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4341d2fb-d4cf-4f9b-b5c4-9e75478fc56f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.380740] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0ee50c8-c81e-49d2-9628-6fb917315a9c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.412906] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20f278af-907c-43f6-951c-738e6242e7d0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.422765] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59cf4a7c-ba64-4c7a-beb1-6bc4c518ba0d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 724.437243] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 724.437243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 4c66ee82ff9042159954bbd5af40fad3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.443752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c66ee82ff9042159954bbd5af40fad3 [ 724.668145] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquiring lock "ee85f964-db45-404c-9016-b618748ec63c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 724.668145] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "ee85f964-db45-404c-9016-b618748ec63c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 724.712262] env[62736]: DEBUG nova.network.neutron [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 724.712262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 0e4ebe2709814ce5bd8639ce2f3049e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.720153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e4ebe2709814ce5bd8639ce2f3049e8 [ 724.746472] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 724.843731] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 724.843731] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg a50c87cf04a24b61b8f861425ac9733d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.852074] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a50c87cf04a24b61b8f861425ac9733d [ 724.943022] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 724.943022] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 68af15b611574aabac7e3b40fd90d32e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 724.954156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68af15b611574aabac7e3b40fd90d32e [ 725.143418] env[62736]: DEBUG nova.compute.manager [req-a003f5c5-2a2d-46a9-923b-e710a0f45d51 req-0d9eb651-3800-47a5-97c8-c6bc15fa117c service nova] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Received event network-vif-deleted-ebf4f464-5af2-4630-82e7-8b0520106de4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 725.216996] env[62736]: INFO nova.compute.manager [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] [instance: 1754ee22-ae95-4439-a6c0-200fd153bad0] Took 1.03 seconds to deallocate network for instance. [ 725.216996] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg c898fbdf1f154a6687d246518dc2c995 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.259284] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c898fbdf1f154a6687d246518dc2c995 [ 725.345405] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 725.345855] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 725.346041] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 725.346344] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3c63a472-ef87-45b7-9240-886095726c40 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 725.355248] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bfe840c-006c-4005-851e-8759361b57f6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 725.375590] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 459055aa-ae56-480f-b2c6-e45cc05bb50c could not be found. [ 725.375804] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 725.375983] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 725.376730] env[62736]: DEBUG oslo.service.loopingcall [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 725.376730] env[62736]: DEBUG nova.compute.manager [-] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 725.376730] env[62736]: DEBUG nova.network.neutron [-] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 725.391212] env[62736]: DEBUG nova.network.neutron [-] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 725.391511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ea70e070fbb741158a4e0c2ea2b8bcab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.398743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea70e070fbb741158a4e0c2ea2b8bcab [ 725.456141] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.473s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 725.456141] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 725.456141] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 7eb01f2ed0c241ecba036b178df2db5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.456141] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.567s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 725.456141] env[62736]: INFO nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 725.457490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 3526fd886b90407a92db9143e7734034 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.494831] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3526fd886b90407a92db9143e7734034 [ 725.499024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7eb01f2ed0c241ecba036b178df2db5c [ 725.722967] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg a150a53abf9f4c24b76164e017d51866 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.759531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a150a53abf9f4c24b76164e017d51866 [ 725.896660] env[62736]: DEBUG nova.network.neutron [-] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 725.897143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 10b14ae26c454e5b8a9111065fd6f50b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.905929] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10b14ae26c454e5b8a9111065fd6f50b [ 725.970494] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 725.971083] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 43c4284d47724cafab37be2331de650c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.973146] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg c26ed1b0b0304553ba8af09f8ee7c7d4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 725.978314] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 725.978314] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 725.983550] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43c4284d47724cafab37be2331de650c [ 725.984044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c26ed1b0b0304553ba8af09f8ee7c7d4 [ 726.016062] env[62736]: DEBUG nova.policy [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c2b0969731654200b597f9d6398c2cfb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '78c69a6dc9124f71b9a1f84fb2da5d4d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 726.247851] env[62736]: INFO nova.scheduler.client.report [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Deleted allocations for instance 1754ee22-ae95-4439-a6c0-200fd153bad0 [ 726.253725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Expecting reply to msg 69c6c374db3744acb09f1e4b097ca2e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 726.267340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69c6c374db3744acb09f1e4b097ca2e9 [ 726.309953] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Successfully created port: 90717c60-aad7-4e97-b134-e7d76933ad99 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 726.399765] env[62736]: INFO nova.compute.manager [-] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Took 1.02 seconds to deallocate network for instance. [ 726.402136] env[62736]: DEBUG nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 726.402343] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.475214] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 726.476961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 934738e7e17548bc8aa4b63207ce4ede in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 726.544987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 934738e7e17548bc8aa4b63207ce4ede [ 726.756044] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8f0ed63-f79a-46cd-8da1-198b55ae3004 tempest-ServersAdminTestJSON-1300144382 tempest-ServersAdminTestJSON-1300144382-project-member] Lock "1754ee22-ae95-4439-a6c0-200fd153bad0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 138.943s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.756644] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 00981f7f4857435ea5ac3c598d8736a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 726.768034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00981f7f4857435ea5ac3c598d8736a8 [ 726.907803] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e1d29e3-6bdd-468d-8ed5-b55cc712273f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.915667] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-872d1dc1-c69f-4bb4-8444-8ab2da1644e7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.946856] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-399f101f-d89b-4a99-b1dd-7e20e33f5dbe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.954345] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-debdb863-4eef-45b7-9d07-8ed76c4ffe97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 726.970068] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 726.970656] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 703ed4e32bc44213852e2ad547f921bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 726.981777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 703ed4e32bc44213852e2ad547f921bf [ 726.983621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg e6de22eb4639489395ee540dc0f531da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.015292] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6de22eb4639489395ee540dc0f531da [ 727.071544] env[62736]: DEBUG nova.compute.manager [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Received event network-changed-90717c60-aad7-4e97-b134-e7d76933ad99 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 727.071844] env[62736]: DEBUG nova.compute.manager [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Refreshing instance network info cache due to event network-changed-90717c60-aad7-4e97-b134-e7d76933ad99. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 727.072091] env[62736]: DEBUG oslo_concurrency.lockutils [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] Acquiring lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 727.072267] env[62736]: DEBUG oslo_concurrency.lockutils [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] Acquired lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 727.072390] env[62736]: DEBUG nova.network.neutron [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Refreshing network info cache for port 90717c60-aad7-4e97-b134-e7d76933ad99 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 727.072816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] Expecting reply to msg 54ed05405c1d488d9ab9b96712feec8f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.079673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54ed05405c1d488d9ab9b96712feec8f [ 727.255181] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 727.255181] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 727.255181] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 727.255181] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 727.255181] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 727.255181] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 727.255181] env[62736]: ERROR nova.compute.manager raise self.value [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 727.255181] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 727.255181] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 727.255181] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 727.255864] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 727.255864] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 727.255864] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 727.255864] env[62736]: ERROR nova.compute.manager [ 727.255864] env[62736]: Traceback (most recent call last): [ 727.255864] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 727.255864] env[62736]: listener.cb(fileno) [ 727.255864] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 727.255864] env[62736]: result = function(*args, **kwargs) [ 727.255864] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 727.255864] env[62736]: return func(*args, **kwargs) [ 727.255864] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 727.255864] env[62736]: raise e [ 727.255864] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 727.255864] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 727.255864] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 727.255864] env[62736]: created_port_ids = self._update_ports_for_instance( [ 727.255864] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 727.255864] env[62736]: with excutils.save_and_reraise_exception(): [ 727.255864] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 727.255864] env[62736]: self.force_reraise() [ 727.255864] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 727.255864] env[62736]: raise self.value [ 727.255864] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 727.255864] env[62736]: updated_port = self._update_port( [ 727.255864] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 727.255864] env[62736]: _ensure_no_port_binding_failure(port) [ 727.255864] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 727.255864] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 727.257000] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 727.257000] env[62736]: Removing descriptor: 16 [ 727.260063] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 727.261864] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 5310cf401d184151a0b06c353ba5fba7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.297203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5310cf401d184151a0b06c353ba5fba7 [ 727.472810] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 727.475234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg b2a4ed007ad84898921e75d9a941f64d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.486512] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 727.490601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2a4ed007ad84898921e75d9a941f64d [ 727.508952] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 727.509189] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 727.509365] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 727.509554] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 727.509698] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 727.509841] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 727.510039] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 727.510189] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 727.510349] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 727.510507] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 727.510674] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 727.511804] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19140e63-38b7-4e8c-8baf-ea586950ac1d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 727.519745] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9038d1c2-198c-453a-80d1-b6204d1ea7fa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 727.534118] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Traceback (most recent call last): [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] yield resources [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.driver.spawn(context, instance, image_meta, [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] vm_ref = self.build_virtual_machine(instance, [ 727.534118] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] vif_infos = vmwarevif.get_vif_info(self._session, [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] for vif in network_info: [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return self._sync_wrapper(fn, *args, **kwargs) [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.wait() [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self[:] = self._gt.wait() [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return self._exit_event.wait() [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 727.534506] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] current.throw(*self._exc) [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] result = function(*args, **kwargs) [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return func(*args, **kwargs) [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise e [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] nwinfo = self.network_api.allocate_for_instance( [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] created_port_ids = self._update_ports_for_instance( [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] with excutils.save_and_reraise_exception(): [ 727.534888] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.force_reraise() [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise self.value [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] updated_port = self._update_port( [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] _ensure_no_port_binding_failure(port) [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise exception.PortBindingFailed(port_id=port['id']) [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 727.535283] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] [ 727.535283] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Terminating instance [ 727.537082] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 727.592339] env[62736]: DEBUG nova.network.neutron [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 727.676374] env[62736]: DEBUG nova.network.neutron [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 727.676939] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] Expecting reply to msg 7005bbc7047c403eb5f633e0b7c65c4a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.685619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7005bbc7047c403eb5f633e0b7c65c4a [ 727.784136] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 727.921349] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "d7947374-0e8f-4d00-a430-923f0f2bb858" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 727.921587] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "d7947374-0e8f-4d00-a430-923f0f2bb858" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 727.978050] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.523s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 727.978448] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 727.980189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 679b031e13344e2a98710e731099133b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 727.981209] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.423s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 727.984260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 5d81bfdd15794a5c9ce64967d4cc5d03 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.013933] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 679b031e13344e2a98710e731099133b [ 728.027810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d81bfdd15794a5c9ce64967d4cc5d03 [ 728.181533] env[62736]: DEBUG oslo_concurrency.lockutils [req-6a25ba68-2dbb-4c84-a239-f19ccc83adc0 req-63c5d85d-8c55-46ad-838d-120d7da360ee service nova] Releasing lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 728.181533] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 728.181533] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 728.181533] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 94760a93fcae4e458bdc896e3bef7c0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.187156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94760a93fcae4e458bdc896e3bef7c0e [ 728.493595] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 728.493595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 549ba9aef3514ae4a2f610b0c29f7cd8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.493595] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 728.493595] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 728.506235] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 549ba9aef3514ae4a2f610b0c29f7cd8 [ 728.596677] env[62736]: DEBUG nova.policy [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c2b0969731654200b597f9d6398c2cfb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '78c69a6dc9124f71b9a1f84fb2da5d4d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 728.710350] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 728.877186] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-020e43f3-cd85-4536-9dc4-0a0c14844f53 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.885997] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d60b37b0-dee5-4738-8a79-2f2b1832dc52 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.920579] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 728.921090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg fb9c7c053117438cb81270ee6ab7609c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.932045] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1dfbe36-3467-49e2-9604-195ad8094783 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.935102] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb9c7c053117438cb81270ee6ab7609c [ 728.941433] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8920aa9-968b-4362-ba64-5d4cf05b399a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 728.960966] env[62736]: DEBUG nova.compute.provider_tree [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 728.961495] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 336fb28427b24f1cb66e21324ac88382 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.969554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 336fb28427b24f1cb66e21324ac88382 [ 728.993609] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 728.995304] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 2a6e17931ef5454893b21d9b55dd1dff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 728.996952] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Successfully created port: 5b2db879-2f10-4314-a5f0-c1f237227a8e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 729.029112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a6e17931ef5454893b21d9b55dd1dff [ 729.112858] env[62736]: DEBUG nova.compute.manager [req-0c7fb91f-1bd4-4df6-80f6-9f6001cda5aa req-1d2b3f5d-6e70-4a7c-96fb-2fc799d0b812 service nova] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Received event network-vif-deleted-90717c60-aad7-4e97-b134-e7d76933ad99 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 729.432184] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 729.432703] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 729.432889] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 729.433183] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4b3f64d8-6b0e-4832-be3a-afb2079b59af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 729.441899] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d202bff3-8412-4aee-a622-f26f9ce569fc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 729.477120] env[62736]: DEBUG nova.scheduler.client.report [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 729.479679] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 5fe9de65c1634dfcb38646b00c563aaf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 729.485270] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5041b2c1-e7f1-4039-b7c4-1f677918025a could not be found. [ 729.485270] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 729.485270] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Took 0.05 seconds to destroy the instance on the hypervisor. [ 729.485270] env[62736]: DEBUG oslo.service.loopingcall [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 729.486098] env[62736]: DEBUG nova.compute.manager [-] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 729.486098] env[62736]: DEBUG nova.network.neutron [-] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 729.493881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fe9de65c1634dfcb38646b00c563aaf [ 729.502304] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg da6a269114284e56b7dd3863525c9822 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 729.512171] env[62736]: DEBUG nova.network.neutron [-] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 729.512626] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 65870c9c657f428c97437825eb088cae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 729.518402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65870c9c657f428c97437825eb088cae [ 729.535101] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da6a269114284e56b7dd3863525c9822 [ 729.920784] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 729.920784] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 729.920784] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 729.920784] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 729.920784] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 729.920784] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 729.920784] env[62736]: ERROR nova.compute.manager raise self.value [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 729.920784] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 729.920784] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 729.920784] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 729.921327] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 729.921327] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 729.921327] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 729.921327] env[62736]: ERROR nova.compute.manager [ 729.921327] env[62736]: Traceback (most recent call last): [ 729.921327] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 729.921327] env[62736]: listener.cb(fileno) [ 729.921327] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 729.921327] env[62736]: result = function(*args, **kwargs) [ 729.921327] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 729.921327] env[62736]: return func(*args, **kwargs) [ 729.921327] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 729.921327] env[62736]: raise e [ 729.921327] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 729.921327] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 729.921327] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 729.921327] env[62736]: created_port_ids = self._update_ports_for_instance( [ 729.921327] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 729.921327] env[62736]: with excutils.save_and_reraise_exception(): [ 729.921327] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 729.921327] env[62736]: self.force_reraise() [ 729.921327] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 729.921327] env[62736]: raise self.value [ 729.921327] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 729.921327] env[62736]: updated_port = self._update_port( [ 729.921327] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 729.921327] env[62736]: _ensure_no_port_binding_failure(port) [ 729.921327] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 729.921327] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 729.922219] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 729.922219] env[62736]: Removing descriptor: 16 [ 729.986965] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.006s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.987771] env[62736]: ERROR nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Traceback (most recent call last): [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.driver.spawn(context, instance, image_meta, [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self._vmops.spawn(context, instance, image_meta, injected_files, [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] vm_ref = self.build_virtual_machine(instance, [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] vif_infos = vmwarevif.get_vif_info(self._session, [ 729.987771] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] for vif in network_info: [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self._sync_wrapper(fn, *args, **kwargs) [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.wait() [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self[:] = self._gt.wait() [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self._exit_event.wait() [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] result = hub.switch() [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 729.988154] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return self.greenlet.switch() [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] result = function(*args, **kwargs) [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] return func(*args, **kwargs) [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise e [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] nwinfo = self.network_api.allocate_for_instance( [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] created_port_ids = self._update_ports_for_instance( [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] with excutils.save_and_reraise_exception(): [ 729.988552] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] self.force_reraise() [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise self.value [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] updated_port = self._update_port( [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] _ensure_no_port_binding_failure(port) [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] raise exception.PortBindingFailed(port_id=port['id']) [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] nova.exception.PortBindingFailed: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. [ 729.988895] env[62736]: ERROR nova.compute.manager [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] [ 729.989250] env[62736]: DEBUG nova.compute.utils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 729.989855] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.199s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.991776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 1f8891708e164ba3827f410f01ce36eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 729.993735] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Build of instance 71ce0048-90fb-460c-a099-7eff5c092d35 was re-scheduled: Binding failed for port 0cfd9e3f-2f54-4f3f-a19a-dbd3ef53fc25, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 729.994239] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 729.994537] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 729.994743] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 729.994956] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 729.995371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg e5d9dd42fd344909be7c6aa599bcf943 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 730.004905] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 730.007247] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5d9dd42fd344909be7c6aa599bcf943 [ 730.014812] env[62736]: DEBUG nova.network.neutron [-] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 730.015389] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 33d0e9c655be4eb4860f46c8fb081218 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 730.025088] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33d0e9c655be4eb4860f46c8fb081218 [ 730.030930] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 730.031231] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 730.031451] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 730.031710] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 730.031943] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 730.032161] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 730.032427] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 730.032646] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 730.032873] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 730.033118] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 730.033379] env[62736]: DEBUG nova.virt.hardware [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 730.034364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f8891708e164ba3827f410f01ce36eb [ 730.035258] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2de275d5-6ae8-4192-a9e7-9e5e0cf9bc5f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.046403] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6f0c21c-3410-4256-99c7-84f47b469344 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.061098] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Traceback (most recent call last): [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] yield resources [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.driver.spawn(context, instance, image_meta, [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] vm_ref = self.build_virtual_machine(instance, [ 730.061098] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] vif_infos = vmwarevif.get_vif_info(self._session, [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] for vif in network_info: [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return self._sync_wrapper(fn, *args, **kwargs) [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.wait() [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self[:] = self._gt.wait() [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return self._exit_event.wait() [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 730.061523] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] current.throw(*self._exc) [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] result = function(*args, **kwargs) [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return func(*args, **kwargs) [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise e [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] nwinfo = self.network_api.allocate_for_instance( [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] created_port_ids = self._update_ports_for_instance( [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] with excutils.save_and_reraise_exception(): [ 730.062086] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.force_reraise() [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise self.value [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] updated_port = self._update_port( [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] _ensure_no_port_binding_failure(port) [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise exception.PortBindingFailed(port_id=port['id']) [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 730.062460] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] [ 730.062460] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Terminating instance [ 730.064518] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 730.064783] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 730.064963] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 730.065418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 0b9e30af97b04ebf9ac56ef67d377e14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 730.071599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b9e30af97b04ebf9ac56ef67d377e14 [ 730.517527] env[62736]: INFO nova.compute.manager [-] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Took 1.03 seconds to deallocate network for instance. [ 730.519873] env[62736]: DEBUG nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 730.520139] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 730.527420] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 730.584470] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 730.667335] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 730.667882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 42dffdfe236347c4bd4f24ab5fdd166b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 730.675999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42dffdfe236347c4bd4f24ab5fdd166b [ 730.683786] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 730.684356] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 68c2251dc91c4d83a7ef67ca1d690777 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 730.698777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68c2251dc91c4d83a7ef67ca1d690777 [ 730.931062] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86f86708-66ea-4cac-b1e1-bb67b8267216 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.950920] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f722ab33-8ae3-468d-ba18-c1ae21c81a94 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.992185] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce926f76-6c72-4937-8481-646d8b41cc19 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 730.999978] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9d65bdd-9437-415e-95f0-6185c1a794f5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.014588] env[62736]: DEBUG nova.compute.provider_tree [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 731.015173] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 0d8d59130df643bea8900ae151ea88cd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.022363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d8d59130df643bea8900ae151ea88cd [ 731.150728] env[62736]: DEBUG nova.compute.manager [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Received event network-changed-5b2db879-2f10-4314-a5f0-c1f237227a8e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 731.150921] env[62736]: DEBUG nova.compute.manager [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Refreshing instance network info cache due to event network-changed-5b2db879-2f10-4314-a5f0-c1f237227a8e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 731.151094] env[62736]: DEBUG oslo_concurrency.lockutils [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] Acquiring lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 731.172617] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-71ce0048-90fb-460c-a099-7eff5c092d35" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 731.172617] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 731.172617] env[62736]: DEBUG nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 731.172617] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 731.187963] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 731.187963] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 731.187963] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 731.187963] env[62736]: DEBUG oslo_concurrency.lockutils [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] Acquired lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 731.187963] env[62736]: DEBUG nova.network.neutron [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Refreshing network info cache for port 5b2db879-2f10-4314-a5f0-c1f237227a8e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 731.188660] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] Expecting reply to msg 29cf21cee6114724b36e8857e8cf9d9b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.189575] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2d119d2e-62de-4096-bbcf-1173b5adbb14 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.196033] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 731.196720] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 4644f9ef1ec142c48cce00b47ff826b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.199225] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29cf21cee6114724b36e8857e8cf9d9b [ 731.202968] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bdaaffa-96db-4055-9879-d6eaff48a348 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 731.215356] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4644f9ef1ec142c48cce00b47ff826b3 [ 731.228926] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 896c2790-9107-4d1c-811b-07a87582c0a1 could not be found. [ 731.229284] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 731.229657] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 731.230027] env[62736]: DEBUG oslo.service.loopingcall [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 731.230389] env[62736]: DEBUG nova.compute.manager [-] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 731.230634] env[62736]: DEBUG nova.network.neutron [-] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 731.245809] env[62736]: DEBUG nova.network.neutron [-] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 731.246571] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 292f090aadea414a8ca52639106d260e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.253057] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 292f090aadea414a8ca52639106d260e [ 731.520948] env[62736]: DEBUG nova.scheduler.client.report [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 731.522480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg c43b8e337eb54b6f8990c4ec74c89292 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.534319] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c43b8e337eb54b6f8990c4ec74c89292 [ 731.700536] env[62736]: DEBUG nova.network.neutron [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.700536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg ae78840c672d4e1a82bd71efdb5f483e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.708580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae78840c672d4e1a82bd71efdb5f483e [ 731.718705] env[62736]: DEBUG nova.network.neutron [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 731.749756] env[62736]: DEBUG nova.network.neutron [-] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.749756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9374abc8948648f2b7023e6e9eb7c4b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.758583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9374abc8948648f2b7023e6e9eb7c4b8 [ 731.825620] env[62736]: DEBUG nova.network.neutron [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.825620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] Expecting reply to msg c541234e91c34078b52e4f0b8d837ec4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 731.835396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c541234e91c34078b52e4f0b8d837ec4 [ 731.985249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4ba35125680c47b8aa74b56a7fdb1075 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.005514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ba35125680c47b8aa74b56a7fdb1075 [ 732.023980] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.034s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.024594] env[62736]: ERROR nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Traceback (most recent call last): [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.driver.spawn(context, instance, image_meta, [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] vm_ref = self.build_virtual_machine(instance, [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] vif_infos = vmwarevif.get_vif_info(self._session, [ 732.024594] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] for vif in network_info: [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self._sync_wrapper(fn, *args, **kwargs) [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.wait() [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self[:] = self._gt.wait() [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self._exit_event.wait() [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] result = hub.switch() [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 732.024994] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return self.greenlet.switch() [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] result = function(*args, **kwargs) [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] return func(*args, **kwargs) [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise e [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] nwinfo = self.network_api.allocate_for_instance( [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] created_port_ids = self._update_ports_for_instance( [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] with excutils.save_and_reraise_exception(): [ 732.025415] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] self.force_reraise() [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise self.value [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] updated_port = self._update_port( [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] _ensure_no_port_binding_failure(port) [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] raise exception.PortBindingFailed(port_id=port['id']) [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] nova.exception.PortBindingFailed: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. [ 732.025866] env[62736]: ERROR nova.compute.manager [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] [ 732.026181] env[62736]: DEBUG nova.compute.utils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 732.026392] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.031s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.028340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 16c943bc26f646dbbc62226f294cfb31 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.030325] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Build of instance 20d9b891-9c2a-479c-8fd8-93405480dee4 was re-scheduled: Binding failed for port 135528e4-b55f-4aeb-a74c-3b85b2005c68, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 732.030772] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 732.031003] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquiring lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 732.031149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Acquired lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 732.031307] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 732.031691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 36634a95e6494de68a740fcd84089528 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.038754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36634a95e6494de68a740fcd84089528 [ 732.071490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16c943bc26f646dbbc62226f294cfb31 [ 732.202400] env[62736]: INFO nova.compute.manager [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 71ce0048-90fb-460c-a099-7eff5c092d35] Took 1.03 seconds to deallocate network for instance. [ 732.204143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 31ad0bcb116f4c8eb9db31fe76f20e75 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.240014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31ad0bcb116f4c8eb9db31fe76f20e75 [ 732.252093] env[62736]: INFO nova.compute.manager [-] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Took 1.02 seconds to deallocate network for instance. [ 732.254428] env[62736]: DEBUG nova.compute.claims [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 732.254612] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.326680] env[62736]: DEBUG oslo_concurrency.lockutils [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] Releasing lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 732.326952] env[62736]: DEBUG nova.compute.manager [req-7a5866c0-1ebd-48d4-9eb1-b686d55e7049 req-f7adf1f7-4e79-4bf6-9837-917287713c6b service nova] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Received event network-vif-deleted-5b2db879-2f10-4314-a5f0-c1f237227a8e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 732.552167] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 732.605790] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 732.606396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 992ad528a6da4949ad20c270d5f556d3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.622936] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 992ad528a6da4949ad20c270d5f556d3 [ 732.709224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 75be93d848ea4d9ba7bbc39c77a12fa2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 732.753136] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75be93d848ea4d9ba7bbc39c77a12fa2 [ 732.977228] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc40ccd8-7ac4-4803-98d0-1589b416003d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 732.984880] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad607cce-2eb7-48ff-8825-10ad08365525 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.025146] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1340a1e-9862-49f4-971b-ff5ab7f7313c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.035279] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bee6f17d-8b18-4ef6-9eb3-15b68ff9b3ec {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.053936] env[62736]: DEBUG nova.compute.provider_tree [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 733.054644] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg b37a521ad97d4800a1b84b2dc9f8461b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.071505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b37a521ad97d4800a1b84b2dc9f8461b [ 733.114440] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Releasing lock "refresh_cache-20d9b891-9c2a-479c-8fd8-93405480dee4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 733.114907] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 733.115059] env[62736]: DEBUG nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 733.115270] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 733.144894] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 733.145457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 9fe17026f4d140e3a800d7406454bbdd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.151950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fe17026f4d140e3a800d7406454bbdd [ 733.233677] env[62736]: INFO nova.scheduler.client.report [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance 71ce0048-90fb-460c-a099-7eff5c092d35 [ 733.250028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 93a6107e623c44878e1aed9887d527d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.262806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93a6107e623c44878e1aed9887d527d1 [ 733.557560] env[62736]: DEBUG nova.scheduler.client.report [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 733.560082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 291db9860a9247599b033615e6dd6014 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.572484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 291db9860a9247599b033615e6dd6014 [ 733.648060] env[62736]: DEBUG nova.network.neutron [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 733.648572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 58fb8d17890d4f06829edf4d436471c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.656333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 58fb8d17890d4f06829edf4d436471c3 [ 733.752534] env[62736]: DEBUG oslo_concurrency.lockutils [None req-d144e025-e9a0-447c-8f76-08532ffe6d50 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "71ce0048-90fb-460c-a099-7eff5c092d35" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.514s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.753125] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg b0615d4e1bbc41909709d6cc4da229d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 733.765526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0615d4e1bbc41909709d6cc4da229d1 [ 734.062776] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.036s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 734.063446] env[62736]: ERROR nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Traceback (most recent call last): [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.driver.spawn(context, instance, image_meta, [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] vm_ref = self.build_virtual_machine(instance, [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] vif_infos = vmwarevif.get_vif_info(self._session, [ 734.063446] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] for vif in network_info: [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self._sync_wrapper(fn, *args, **kwargs) [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.wait() [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self[:] = self._gt.wait() [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self._exit_event.wait() [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] result = hub.switch() [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 734.063805] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return self.greenlet.switch() [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] result = function(*args, **kwargs) [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] return func(*args, **kwargs) [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise e [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] nwinfo = self.network_api.allocate_for_instance( [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] created_port_ids = self._update_ports_for_instance( [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] with excutils.save_and_reraise_exception(): [ 734.064205] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] self.force_reraise() [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise self.value [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] updated_port = self._update_port( [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] _ensure_no_port_binding_failure(port) [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] raise exception.PortBindingFailed(port_id=port['id']) [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] nova.exception.PortBindingFailed: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. [ 734.064563] env[62736]: ERROR nova.compute.manager [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] [ 734.064916] env[62736]: DEBUG nova.compute.utils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 734.065567] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.788s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 734.066971] env[62736]: INFO nova.compute.claims [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 734.069007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg f3afa67e9b0040bd85a71bd739f96c2c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.070660] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Build of instance 48a0d4f8-4fab-454b-b065-e552647fba3f was re-scheduled: Binding failed for port 749951e6-e808-4d96-b4e6-bbcce0326a32, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 734.071103] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 734.071564] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquiring lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 734.071564] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Acquired lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 734.071831] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 734.073441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 53d92bc1aea84a95b82e18d08007a381 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.080111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53d92bc1aea84a95b82e18d08007a381 [ 734.119719] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3afa67e9b0040bd85a71bd739f96c2c [ 734.151149] env[62736]: INFO nova.compute.manager [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] [instance: 20d9b891-9c2a-479c-8fd8-93405480dee4] Took 1.04 seconds to deallocate network for instance. [ 734.152850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg b9db3ae8099e4b59bec9a61d1341a4bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.189318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9db3ae8099e4b59bec9a61d1341a4bb [ 734.255316] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 734.257761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 72b7f0e269084afb8e55b9785591fd4b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.291360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72b7f0e269084afb8e55b9785591fd4b [ 734.577361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 54707efd172b46d79f2091601bea4d6e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.585994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54707efd172b46d79f2091601bea4d6e [ 734.595235] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 734.657372] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 5002023b6e3143878fa398c0de36d074 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.698648] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5002023b6e3143878fa398c0de36d074 [ 734.714553] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 734.715084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 950c888dc3b94249969b2dcca1d8f505 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 734.722850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 950c888dc3b94249969b2dcca1d8f505 [ 734.778533] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 734.951155] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquiring lock "76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 734.951520] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.196193] env[62736]: INFO nova.scheduler.client.report [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Deleted allocations for instance 20d9b891-9c2a-479c-8fd8-93405480dee4 [ 735.203321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Expecting reply to msg 61316e51dba94f38b8d204d7c592520f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 735.217785] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Releasing lock "refresh_cache-48a0d4f8-4fab-454b-b065-e552647fba3f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 735.218007] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 735.218181] env[62736]: DEBUG nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 735.218339] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 735.220589] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61316e51dba94f38b8d204d7c592520f [ 735.247928] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 735.248754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 9074c80841ea410aaad9fb791b45b974 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 735.260140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9074c80841ea410aaad9fb791b45b974 [ 735.459403] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1aeccc75-3a36-4b72-b860-de8bbaabb9af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.467598] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04540a8e-a79f-47c3-923d-5b26bfb12127 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.504309] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-266f821a-ab1a-4a39-9b75-7fcbc931e30f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.511814] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e7fb09b-6bb4-4655-9751-d7736ce67e77 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.529713] env[62736]: DEBUG nova.compute.provider_tree [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 735.530338] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 1b7120942a8546ebb90e9cfb0c1db527 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 735.539112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b7120942a8546ebb90e9cfb0c1db527 [ 735.706231] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c7344060-aed2-469b-b21e-91e68c5e9f58 tempest-MigrationsAdminTest-791669259 tempest-MigrationsAdminTest-791669259-project-member] Lock "20d9b891-9c2a-479c-8fd8-93405480dee4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.340s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.706954] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 8d1a4156384b4355aea14df3daeef6fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 735.724515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d1a4156384b4355aea14df3daeef6fc [ 735.754477] env[62736]: DEBUG nova.network.neutron [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 735.754477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 579a4d249fee43a18ed1ea2d56a06ca0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 735.765699] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 579a4d249fee43a18ed1ea2d56a06ca0 [ 735.802688] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "d866dcf0-c08b-4b12-be83-8de3e69c439f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.802984] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "d866dcf0-c08b-4b12-be83-8de3e69c439f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 736.036047] env[62736]: DEBUG nova.scheduler.client.report [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 736.036681] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 518fa0b01cd6495c85bd5e847c1f1a51 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.047468] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 518fa0b01cd6495c85bd5e847c1f1a51 [ 736.209791] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 736.211572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 372f1a12bdfe40379f37081c6e3eb809 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.245479] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 372f1a12bdfe40379f37081c6e3eb809 [ 736.256308] env[62736]: INFO nova.compute.manager [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] [instance: 48a0d4f8-4fab-454b-b065-e552647fba3f] Took 1.04 seconds to deallocate network for instance. [ 736.258102] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 30eedba61fd14e55bb4a779c5a82a6a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.297521] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30eedba61fd14e55bb4a779c5a82a6a1 [ 736.540199] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.474s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 736.540560] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 736.542665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 052c905fca784c5a90b15392baa113de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.543176] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.347s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 736.544623] env[62736]: INFO nova.compute.claims [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 736.550501] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 251137a4b25c4b0eb853f98bd24301a2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.575989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 052c905fca784c5a90b15392baa113de [ 736.593912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 251137a4b25c4b0eb853f98bd24301a2 [ 736.731685] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 736.763914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg c6c520a34d7843eab7dc96db2b32a71e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 736.805733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6c520a34d7843eab7dc96db2b32a71e [ 737.049529] env[62736]: DEBUG nova.compute.utils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 737.050183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 6e8aeab8cb2d42b396287e9fc25adfb4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 737.051080] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 737.051225] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 737.058755] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 29112e9b1a4d4c249d49c58d43fa2306 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 737.061617] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e8aeab8cb2d42b396287e9fc25adfb4 [ 737.066333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29112e9b1a4d4c249d49c58d43fa2306 [ 737.126808] env[62736]: DEBUG nova.policy [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '717a6eb5643b4f188ec28201b885675e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f160fdbb9ae34ba3a4a17c903121bc49', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 737.290460] env[62736]: INFO nova.scheduler.client.report [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Deleted allocations for instance 48a0d4f8-4fab-454b-b065-e552647fba3f [ 737.303251] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Expecting reply to msg 233a8d5662144acbac0616f0b214fae9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 737.334824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 233a8d5662144acbac0616f0b214fae9 [ 737.513315] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Successfully created port: 28401104-aeb3-4305-8c00-72354272efa2 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 737.555148] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 737.556925] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 92f8e7c7ec9c495482c3c5d5379be110 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 737.630038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92f8e7c7ec9c495482c3c5d5379be110 [ 737.798456] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7428bf23-5284-44f0-bc6a-2c44248c37fe tempest-AttachInterfacesUnderV243Test-1451027736 tempest-AttachInterfacesUnderV243Test-1451027736-project-member] Lock "48a0d4f8-4fab-454b-b065-e552647fba3f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 144.655s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 737.799037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 9c6124aab88e4e09b4c5aa041a927df2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 737.814197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c6124aab88e4e09b4c5aa041a927df2 [ 737.942654] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28a7fb83-e90a-47ad-afb6-888759b65d59 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.950943] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a6190cc-b700-4409-b4f2-b34930e5009b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.982245] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45f28e1c-d4d4-40c0-b705-fba77bb9299f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 737.990357] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5445b477-b725-48cd-8b6e-73808c75e744 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.003176] env[62736]: DEBUG nova.compute.provider_tree [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 738.003683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg da186bb206804b969da6ce41b7209775 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 738.011441] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da186bb206804b969da6ce41b7209775 [ 738.061880] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg a661cccea6754218bd8628108bb6ec1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 738.095652] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a661cccea6754218bd8628108bb6ec1d [ 738.301548] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 738.303296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 587efb53e9794cd4a90fb47e20ff0ea0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 738.342662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 587efb53e9794cd4a90fb47e20ff0ea0 [ 738.507040] env[62736]: DEBUG nova.scheduler.client.report [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 738.509649] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 039d723142bb42d7ba5a1a7c7a8e0468 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 738.521595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 039d723142bb42d7ba5a1a7c7a8e0468 [ 738.565426] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 738.587818] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 738.588084] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 738.588245] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 738.588423] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 738.588564] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 738.588705] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 738.588902] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 738.589053] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 738.589212] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 738.589371] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 738.589547] env[62736]: DEBUG nova.virt.hardware [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 738.590399] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62ccb6bf-7e2a-40ce-904d-9541ea378553 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.598350] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2f9e462-a36e-4276-8216-6e7ceb4551d0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.764851] env[62736]: DEBUG nova.compute.manager [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Received event network-changed-28401104-aeb3-4305-8c00-72354272efa2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 738.765060] env[62736]: DEBUG nova.compute.manager [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Refreshing instance network info cache due to event network-changed-28401104-aeb3-4305-8c00-72354272efa2. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 738.765275] env[62736]: DEBUG oslo_concurrency.lockutils [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] Acquiring lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 738.765413] env[62736]: DEBUG oslo_concurrency.lockutils [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] Acquired lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 738.765568] env[62736]: DEBUG nova.network.neutron [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Refreshing network info cache for port 28401104-aeb3-4305-8c00-72354272efa2 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 738.765990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] Expecting reply to msg 9769b95c348846cc940cd06ce4bf4f8d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 738.773662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9769b95c348846cc940cd06ce4bf4f8d [ 738.823969] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.986326] env[62736]: ERROR nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 738.986326] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 738.986326] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 738.986326] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 738.986326] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 738.986326] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 738.986326] env[62736]: ERROR nova.compute.manager raise self.value [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 738.986326] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 738.986326] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 738.986326] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 738.986875] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 738.986875] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 738.986875] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 738.986875] env[62736]: ERROR nova.compute.manager [ 738.986875] env[62736]: Traceback (most recent call last): [ 738.986875] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 738.986875] env[62736]: listener.cb(fileno) [ 738.986875] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 738.986875] env[62736]: result = function(*args, **kwargs) [ 738.986875] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 738.986875] env[62736]: return func(*args, **kwargs) [ 738.986875] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 738.986875] env[62736]: raise e [ 738.986875] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 738.986875] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 738.986875] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 738.986875] env[62736]: created_port_ids = self._update_ports_for_instance( [ 738.986875] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 738.986875] env[62736]: with excutils.save_and_reraise_exception(): [ 738.986875] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 738.986875] env[62736]: self.force_reraise() [ 738.986875] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 738.986875] env[62736]: raise self.value [ 738.986875] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 738.986875] env[62736]: updated_port = self._update_port( [ 738.986875] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 738.986875] env[62736]: _ensure_no_port_binding_failure(port) [ 738.986875] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 738.986875] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 738.989061] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 738.989061] env[62736]: Removing descriptor: 17 [ 738.989061] env[62736]: ERROR nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Traceback (most recent call last): [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] yield resources [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.driver.spawn(context, instance, image_meta, [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self._vmops.spawn(context, instance, image_meta, injected_files, [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 738.989061] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] vm_ref = self.build_virtual_machine(instance, [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] vif_infos = vmwarevif.get_vif_info(self._session, [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] for vif in network_info: [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self._sync_wrapper(fn, *args, **kwargs) [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.wait() [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self[:] = self._gt.wait() [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self._exit_event.wait() [ 738.989530] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] result = hub.switch() [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self.greenlet.switch() [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] result = function(*args, **kwargs) [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return func(*args, **kwargs) [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise e [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] nwinfo = self.network_api.allocate_for_instance( [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 738.989963] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] created_port_ids = self._update_ports_for_instance( [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] with excutils.save_and_reraise_exception(): [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.force_reraise() [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise self.value [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] updated_port = self._update_port( [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] _ensure_no_port_binding_failure(port) [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 738.990380] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise exception.PortBindingFailed(port_id=port['id']) [ 738.990892] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 738.990892] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] [ 738.990892] env[62736]: INFO nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Terminating instance [ 738.990892] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquiring lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 739.016493] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.469s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 739.016493] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 739.016493] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 0dc7925608154c65ae087034a6ee3d74 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.016493] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.744s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 739.025032] env[62736]: INFO nova.compute.claims [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 739.026714] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 5968046b79de4d5b912675370dc6e0d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.052322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0dc7925608154c65ae087034a6ee3d74 [ 739.080248] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5968046b79de4d5b912675370dc6e0d5 [ 739.298893] env[62736]: DEBUG nova.network.neutron [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 739.466347] env[62736]: DEBUG nova.network.neutron [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 739.466876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] Expecting reply to msg 3f65158ff09a437a9f3e86500ffdd36e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.475255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f65158ff09a437a9f3e86500ffdd36e [ 739.530182] env[62736]: DEBUG nova.compute.utils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 739.530821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg b31bb98d776e4ef2b058047e26dd35fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.532961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg ad6192477ab140dcbe0d7cbba5d4d1c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.540391] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 739.540570] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 739.542628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b31bb98d776e4ef2b058047e26dd35fc [ 739.545835] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad6192477ab140dcbe0d7cbba5d4d1c7 [ 739.629850] env[62736]: DEBUG nova.policy [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e5776ada81cb4526966ec39aad15fc4f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '490a56a64a044ddf955e8c0056890140', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 739.976126] env[62736]: DEBUG oslo_concurrency.lockutils [req-a5bf949a-51db-4c6c-853d-06901be3c02b req-94c9826b-7a24-4c91-b984-924d88c9edf8 service nova] Releasing lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 739.976508] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquired lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 739.976685] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 739.977116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg d7a8ef96923145faa3b22e9652a1bf15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 739.983719] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7a8ef96923145faa3b22e9652a1bf15 [ 740.041105] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 740.042913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 34ffaf642b694a73a56e21c4d2cc4939 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.085673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 34ffaf642b694a73a56e21c4d2cc4939 [ 740.327983] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Successfully created port: 701fcdef-8298-4a8d-8739-5a629c9d99e4 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 740.391736] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29f08e5d-12af-451c-959a-d8e24146de3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.413581] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9201486-1e99-45b1-b4e2-9eb71442ddc7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.452911] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0959e7b7-9452-4230-98e5-e85c4e4abba9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.460851] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62aa30ed-7d25-461a-8b79-4884dfd39fca {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.475079] env[62736]: DEBUG nova.compute.provider_tree [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 740.475574] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 99eaeeb04c664a76a1725d2655b4d654 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.485155] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99eaeeb04c664a76a1725d2655b4d654 [ 740.502694] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 740.565343] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 9aef95414b89458897095052d2a88316 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.599224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9aef95414b89458897095052d2a88316 [ 740.609144] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 740.609660] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg e6d8d1beb385447ab67c71567187e47e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.619504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6d8d1beb385447ab67c71567187e47e [ 740.620257] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Releasing lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 740.620614] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 740.620800] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 740.621328] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-184e6389-059c-4dc0-ab5b-b6fe4b975aa0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.633225] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d53e8d52-86c7-49c8-a1cb-978ee4fffa22 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 740.659193] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1174dcea-6ba6-4a2b-9eb4-85e662691c30 could not be found. [ 740.659593] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 740.659690] env[62736]: INFO nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Took 0.04 seconds to destroy the instance on the hypervisor. [ 740.659880] env[62736]: DEBUG oslo.service.loopingcall [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 740.660291] env[62736]: DEBUG nova.compute.manager [-] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 740.660389] env[62736]: DEBUG nova.network.neutron [-] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 740.679275] env[62736]: DEBUG nova.network.neutron [-] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 740.679928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b81713dfa0db4e648c016ead03f3bb2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.701008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b81713dfa0db4e648c016ead03f3bb2b [ 740.984622] env[62736]: DEBUG nova.scheduler.client.report [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 740.987028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg bb5f7a526ad8494f858589d4416725c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 740.997418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb5f7a526ad8494f858589d4416725c9 [ 741.033175] env[62736]: DEBUG nova.compute.manager [req-e9f2f92a-d14c-4e94-9ea6-8682dfdd4ef5 req-d8fa6b68-8329-4ba5-acef-667b334adedc service nova] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Received event network-vif-deleted-28401104-aeb3-4305-8c00-72354272efa2 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 741.063101] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 741.089676] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 741.089935] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 741.090088] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 741.090259] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 741.090427] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 741.090601] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 741.090806] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 741.090962] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 741.091119] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 741.091275] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 741.091441] env[62736]: DEBUG nova.virt.hardware [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 741.092307] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4900451e-ffe7-4327-8dbe-7e4079ba8142 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.100837] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d24af5d7-7d37-47a4-be75-639d7cc1a634 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 741.182880] env[62736]: DEBUG nova.network.neutron [-] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 741.183531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1e6f3e2fdd9040aabf06d0f9954a6ce7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 741.191664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e6f3e2fdd9040aabf06d0f9954a6ce7 [ 741.492442] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.476s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.492942] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 741.494753] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg e22e676c6ffd4301a5f49c7aabaaafe5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 741.495776] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.093s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 741.497462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 807fdda3895f4656badd813636ba0ae5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 741.536394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e22e676c6ffd4301a5f49c7aabaaafe5 [ 741.541947] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 807fdda3895f4656badd813636ba0ae5 [ 741.685929] env[62736]: INFO nova.compute.manager [-] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Took 1.03 seconds to deallocate network for instance. [ 741.688840] env[62736]: DEBUG nova.compute.claims [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 741.689041] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 741.823814] env[62736]: ERROR nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 741.823814] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 741.823814] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 741.823814] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 741.823814] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 741.823814] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 741.823814] env[62736]: ERROR nova.compute.manager raise self.value [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 741.823814] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 741.823814] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 741.823814] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 741.824625] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 741.824625] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 741.824625] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 741.824625] env[62736]: ERROR nova.compute.manager [ 741.824625] env[62736]: Traceback (most recent call last): [ 741.824625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 741.824625] env[62736]: listener.cb(fileno) [ 741.824625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 741.824625] env[62736]: result = function(*args, **kwargs) [ 741.824625] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 741.824625] env[62736]: return func(*args, **kwargs) [ 741.824625] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 741.824625] env[62736]: raise e [ 741.824625] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 741.824625] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 741.824625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 741.824625] env[62736]: created_port_ids = self._update_ports_for_instance( [ 741.824625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 741.824625] env[62736]: with excutils.save_and_reraise_exception(): [ 741.824625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 741.824625] env[62736]: self.force_reraise() [ 741.824625] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 741.824625] env[62736]: raise self.value [ 741.824625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 741.824625] env[62736]: updated_port = self._update_port( [ 741.824625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 741.824625] env[62736]: _ensure_no_port_binding_failure(port) [ 741.824625] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 741.824625] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 741.826111] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 741.826111] env[62736]: Removing descriptor: 17 [ 741.826111] env[62736]: ERROR nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Traceback (most recent call last): [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] yield resources [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.driver.spawn(context, instance, image_meta, [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self._vmops.spawn(context, instance, image_meta, injected_files, [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 741.826111] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] vm_ref = self.build_virtual_machine(instance, [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] vif_infos = vmwarevif.get_vif_info(self._session, [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] for vif in network_info: [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self._sync_wrapper(fn, *args, **kwargs) [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.wait() [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self[:] = self._gt.wait() [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self._exit_event.wait() [ 741.826707] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] result = hub.switch() [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self.greenlet.switch() [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] result = function(*args, **kwargs) [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return func(*args, **kwargs) [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise e [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] nwinfo = self.network_api.allocate_for_instance( [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 741.827391] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] created_port_ids = self._update_ports_for_instance( [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] with excutils.save_and_reraise_exception(): [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.force_reraise() [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise self.value [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] updated_port = self._update_port( [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] _ensure_no_port_binding_failure(port) [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 741.828382] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise exception.PortBindingFailed(port_id=port['id']) [ 741.829248] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 741.829248] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] [ 741.829248] env[62736]: INFO nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Terminating instance [ 741.829248] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquiring lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 741.829248] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquired lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 741.829248] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 741.829583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 514944b3e1ae4113afb9b27ba685f680 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 741.835371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 514944b3e1ae4113afb9b27ba685f680 [ 742.010555] env[62736]: DEBUG nova.compute.utils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 742.011200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg ec69eef9a7a74c78a36650d5972ce0f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.015601] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 742.015815] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 742.021614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec69eef9a7a74c78a36650d5972ce0f7 [ 742.083390] env[62736]: DEBUG nova.policy [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '43c10ef2ec3c40ea985d9d1213f25e6b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dab569c547cb41cea49d87164ed92136', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 742.283692] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.283941] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.292411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3f5fae577c4c48408a2c2c65138ee5e2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.307017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f5fae577c4c48408a2c2c65138ee5e2 [ 742.356333] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 742.390155] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4216c600-23e6-40fe-bd46-126134c5155e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.398183] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e909a730-42d6-45ee-a82d-a6e391dda338 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.427985] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd8f3f7f-a177-4f47-ad31-1e30bfbab3c4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.435334] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2907e2b4-31cc-46cd-925b-76d3bc49e985 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.448832] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 742.449244] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 7fea1ddd2d844be7a16de1cd7dd7ea16 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.456604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fea1ddd2d844be7a16de1cd7dd7ea16 [ 742.516123] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 742.518115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg e5b91e3c6e7e47c5b7ca2c41803613a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.536503] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 742.539293] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg ebe9a653a19c460595e4142f06b842bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.557865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5b91e3c6e7e47c5b7ca2c41803613a3 [ 742.559375] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebe9a653a19c460595e4142f06b842bb [ 742.802277] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 742.802465] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 742.802614] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 742.803286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 74bc9f0757d74631ba97b6bd8c636995 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.825451] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74bc9f0757d74631ba97b6bd8c636995 [ 742.894309] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Successfully created port: cb3537e6-adcb-4b16-9fc1-bb454d4e1c56 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 742.952512] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 742.954815] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 8d58147ed893446d9bb77a47094094de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 742.970960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d58147ed893446d9bb77a47094094de [ 743.022996] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 482d1a491f97479b9e25e5676d2fc422 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.053715] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Releasing lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 743.053715] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 743.053715] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 743.053715] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-a39ec12c-51af-4bd9-b6dd-3dfd8f39037a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.059976] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-540acf1f-26e7-4ae0-9de1-e3a904c7ff1b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.072317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 482d1a491f97479b9e25e5676d2fc422 [ 743.083109] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1a5a4c2e-74e6-4a62-adda-c3d89deecf05 could not be found. [ 743.083449] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 743.083878] env[62736]: INFO nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Took 0.03 seconds to destroy the instance on the hypervisor. [ 743.084315] env[62736]: DEBUG oslo.service.loopingcall [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 743.084648] env[62736]: DEBUG nova.compute.manager [-] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 743.084937] env[62736]: DEBUG nova.network.neutron [-] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 743.117220] env[62736]: DEBUG nova.network.neutron [-] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 743.117914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg cbe2e44402114d1998bf4638f8278cd3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.131194] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbe2e44402114d1998bf4638f8278cd3 [ 743.144510] env[62736]: DEBUG nova.compute.manager [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Received event network-changed-701fcdef-8298-4a8d-8739-5a629c9d99e4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 743.144862] env[62736]: DEBUG nova.compute.manager [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Refreshing instance network info cache due to event network-changed-701fcdef-8298-4a8d-8739-5a629c9d99e4. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 743.145291] env[62736]: DEBUG oslo_concurrency.lockutils [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] Acquiring lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 743.145558] env[62736]: DEBUG oslo_concurrency.lockutils [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] Acquired lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 743.145825] env[62736]: DEBUG nova.network.neutron [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Refreshing network info cache for port 701fcdef-8298-4a8d-8739-5a629c9d99e4 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 743.146327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] Expecting reply to msg 2fa11797f47a43289fe308d79de1af99 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.154271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2fa11797f47a43289fe308d79de1af99 [ 743.307849] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 743.307849] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 743.307963] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 743.308021] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 743.308140] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 743.308261] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 743.308457] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.308601] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.308737] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.308874] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.309007] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.309143] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.309261] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 743.309423] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 743.309780] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b67c664a50be488286a28caed7dd4b9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.321271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b67c664a50be488286a28caed7dd4b9f [ 743.457879] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.962s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 743.458850] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Traceback (most recent call last): [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.driver.spawn(context, instance, image_meta, [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] vm_ref = self.build_virtual_machine(instance, [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] vif_infos = vmwarevif.get_vif_info(self._session, [ 743.458850] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] for vif in network_info: [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self._sync_wrapper(fn, *args, **kwargs) [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.wait() [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self[:] = self._gt.wait() [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self._exit_event.wait() [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] result = hub.switch() [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 743.459238] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return self.greenlet.switch() [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] result = function(*args, **kwargs) [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] return func(*args, **kwargs) [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise e [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] nwinfo = self.network_api.allocate_for_instance( [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] created_port_ids = self._update_ports_for_instance( [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] with excutils.save_and_reraise_exception(): [ 743.459686] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] self.force_reraise() [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise self.value [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] updated_port = self._update_port( [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] _ensure_no_port_binding_failure(port) [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] raise exception.PortBindingFailed(port_id=port['id']) [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] nova.exception.PortBindingFailed: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. [ 743.460086] env[62736]: ERROR nova.compute.manager [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] [ 743.460406] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 743.461056] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.677s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 743.463685] env[62736]: INFO nova.compute.claims [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 743.466357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 7c5d4bdd59dc43299ea9138df651b15d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.468187] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Build of instance 459055aa-ae56-480f-b2c6-e45cc05bb50c was re-scheduled: Binding failed for port ebf4f464-5af2-4630-82e7-8b0520106de4, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 743.468863] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 743.469275] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 743.470218] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 743.470403] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 743.470959] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg a100f51bdaaf41aa8e32f2d52ff4109b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.482682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a100f51bdaaf41aa8e32f2d52ff4109b [ 743.512367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c5d4bdd59dc43299ea9138df651b15d [ 743.539598] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 743.564798] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 743.565044] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 743.565208] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 743.565368] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 743.565510] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 743.565654] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 743.565851] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 743.566004] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 743.566165] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 743.566586] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 743.566820] env[62736]: DEBUG nova.virt.hardware [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 743.567820] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28dc8919-0ab6-4049-a525-3f5c600f8583 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.576401] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3246cf0d-84f2-455a-83f6-d46a2dd32a24 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.626746] env[62736]: DEBUG nova.network.neutron [-] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.627252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e42e7b01deaf4a1a90d01564dc091a2c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.659837] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e42e7b01deaf4a1a90d01564dc091a2c [ 743.672439] env[62736]: DEBUG nova.network.neutron [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 743.812791] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 743.961934] env[62736]: DEBUG nova.network.neutron [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.962464] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] Expecting reply to msg 78f6a0e1cbdb49f3bc96d6d4ca48caa3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.970348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg ce2da35094a448dc96b948e93bea81bc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 743.971359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78f6a0e1cbdb49f3bc96d6d4ca48caa3 [ 743.980520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce2da35094a448dc96b948e93bea81bc [ 744.032343] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 744.129824] env[62736]: INFO nova.compute.manager [-] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Took 1.04 seconds to deallocate network for instance. [ 744.132215] env[62736]: DEBUG nova.compute.claims [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 744.132392] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.205523] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 744.206075] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg d4637e6076fb4b88a485d5707fcb3742 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 744.214875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4637e6076fb4b88a485d5707fcb3742 [ 744.464869] env[62736]: DEBUG oslo_concurrency.lockutils [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] Releasing lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 744.465169] env[62736]: DEBUG nova.compute.manager [req-e3fcf756-a229-4a96-9672-88bf43f52028 req-77052dda-a172-472c-bf2c-8dbb592115e0 service nova] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Received event network-vif-deleted-701fcdef-8298-4a8d-8739-5a629c9d99e4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 744.561529] env[62736]: ERROR nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 744.561529] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.561529] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 744.561529] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 744.561529] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.561529] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.561529] env[62736]: ERROR nova.compute.manager raise self.value [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 744.561529] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 744.561529] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.561529] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 744.562016] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.562016] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 744.562016] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 744.562016] env[62736]: ERROR nova.compute.manager [ 744.562016] env[62736]: Traceback (most recent call last): [ 744.562016] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 744.562016] env[62736]: listener.cb(fileno) [ 744.562016] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 744.562016] env[62736]: result = function(*args, **kwargs) [ 744.562016] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 744.562016] env[62736]: return func(*args, **kwargs) [ 744.562016] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 744.562016] env[62736]: raise e [ 744.562016] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.562016] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 744.562016] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 744.562016] env[62736]: created_port_ids = self._update_ports_for_instance( [ 744.562016] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 744.562016] env[62736]: with excutils.save_and_reraise_exception(): [ 744.562016] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.562016] env[62736]: self.force_reraise() [ 744.562016] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.562016] env[62736]: raise self.value [ 744.562016] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 744.562016] env[62736]: updated_port = self._update_port( [ 744.562016] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.562016] env[62736]: _ensure_no_port_binding_failure(port) [ 744.562016] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.562016] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 744.562871] env[62736]: nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 744.562871] env[62736]: Removing descriptor: 17 [ 744.562995] env[62736]: ERROR nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Traceback (most recent call last): [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] yield resources [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.driver.spawn(context, instance, image_meta, [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] vm_ref = self.build_virtual_machine(instance, [ 744.562995] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] vif_infos = vmwarevif.get_vif_info(self._session, [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] for vif in network_info: [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self._sync_wrapper(fn, *args, **kwargs) [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.wait() [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self[:] = self._gt.wait() [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self._exit_event.wait() [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 744.563346] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] result = hub.switch() [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self.greenlet.switch() [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] result = function(*args, **kwargs) [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return func(*args, **kwargs) [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise e [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] nwinfo = self.network_api.allocate_for_instance( [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] created_port_ids = self._update_ports_for_instance( [ 744.563739] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] with excutils.save_and_reraise_exception(): [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.force_reraise() [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise self.value [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] updated_port = self._update_port( [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] _ensure_no_port_binding_failure(port) [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise exception.PortBindingFailed(port_id=port['id']) [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 744.564138] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] [ 744.564528] env[62736]: INFO nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Terminating instance [ 744.567828] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquiring lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 744.568080] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquired lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 744.568318] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 744.568789] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 00fd4910a41f4bd88fac406089138a0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 744.576233] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00fd4910a41f4bd88fac406089138a0e [ 744.708248] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-459055aa-ae56-480f-b2c6-e45cc05bb50c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 744.708541] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 744.708770] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 744.708974] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 744.726982] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 744.727575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 178dd67ddd1f441d9eed76d0bc39f302 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 744.736733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 178dd67ddd1f441d9eed76d0bc39f302 [ 744.883734] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9a4d40e-d56c-42b3-8c49-c13468d6feed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.891917] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11a0b2c6-8aba-4940-a6e7-fee8f8684842 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.930351] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f99bee8-3afa-439a-ad53-b6db797382c5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.937730] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25d13d35-2cc0-4aff-aa1a-b5e364c86853 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.950862] env[62736]: DEBUG nova.compute.provider_tree [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 744.951549] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 478139ae42e148359a33b696d2bf0298 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 744.960172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 478139ae42e148359a33b696d2bf0298 [ 745.094809] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 745.239111] env[62736]: DEBUG nova.compute.manager [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Received event network-changed-cb3537e6-adcb-4b16-9fc1-bb454d4e1c56 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 745.239111] env[62736]: DEBUG nova.compute.manager [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Refreshing instance network info cache due to event network-changed-cb3537e6-adcb-4b16-9fc1-bb454d4e1c56. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 745.239111] env[62736]: DEBUG oslo_concurrency.lockutils [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] Acquiring lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 745.239111] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 745.239111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 63cf061e98c5420398ec052665cc6459 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.250164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63cf061e98c5420398ec052665cc6459 [ 745.261991] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 745.262520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 4cbcd14c363d4053b7641081f7471ded in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.277431] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4cbcd14c363d4053b7641081f7471ded [ 745.453950] env[62736]: DEBUG nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 745.456425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg a7cfc3e5577a45fcb7a965a633a7defd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.468914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7cfc3e5577a45fcb7a965a633a7defd [ 745.745225] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 459055aa-ae56-480f-b2c6-e45cc05bb50c] Took 1.04 seconds to deallocate network for instance. [ 745.746984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 1c65652398ec46d5be58953dbf726b2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.764705] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Releasing lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 745.765151] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 745.765342] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 745.765768] env[62736]: DEBUG oslo_concurrency.lockutils [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] Acquired lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 745.765978] env[62736]: DEBUG nova.network.neutron [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Refreshing network info cache for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 745.766401] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] Expecting reply to msg 6817a455b1884849bcb58a85b8d83eff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.767180] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-065a1d45-d119-48e4-9788-0d70c653244c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.774888] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6817a455b1884849bcb58a85b8d83eff [ 745.778220] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e05890e7-c970-40c5-bc4d-94259a390c71 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.801629] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 427400bc-f265-40cb-8c1c-ccab7bc94b1c could not be found. [ 745.801871] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 745.802038] env[62736]: INFO nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Took 0.04 seconds to destroy the instance on the hypervisor. [ 745.802528] env[62736]: DEBUG oslo.service.loopingcall [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 745.802528] env[62736]: DEBUG nova.compute.manager [-] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 745.802655] env[62736]: DEBUG nova.network.neutron [-] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 745.811754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c65652398ec46d5be58953dbf726b2d [ 745.839555] env[62736]: DEBUG nova.network.neutron [-] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 745.840258] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b6f5a5b702d642d3bbf4244fb9926d19 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.846931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6f5a5b702d642d3bbf4244fb9926d19 [ 745.958682] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.498s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 745.959251] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 745.960988] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 90b1c2fefec84ceab1386f42ca9601b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.962039] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.442s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 745.963768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 101921b0c6a54872a8f22451b2e16a95 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 745.998115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90b1c2fefec84ceab1386f42ca9601b4 [ 746.006651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 101921b0c6a54872a8f22451b2e16a95 [ 746.252390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 0e4592e130fa4107a8d2792522c3023c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.286103] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e4592e130fa4107a8d2792522c3023c [ 746.310402] env[62736]: DEBUG nova.network.neutron [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 746.342170] env[62736]: DEBUG nova.network.neutron [-] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 746.342637] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ce02a5f78789449591eddbf7432e5838 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.361772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce02a5f78789449591eddbf7432e5838 [ 746.423739] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "ddcf62db-af40-49d2-9465-66a10f0f2e2d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.423739] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "ddcf62db-af40-49d2-9465-66a10f0f2e2d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.449020] env[62736]: DEBUG nova.network.neutron [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 746.449020] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] Expecting reply to msg 0bc1ce31ef54424caa89d38ede0edb81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.456566] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bc1ce31ef54424caa89d38ede0edb81 [ 746.470589] env[62736]: DEBUG nova.compute.utils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 746.471163] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 34529f3ec0714f109644f51875048d2e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.472288] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 746.472429] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 746.487301] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 34529f3ec0714f109644f51875048d2e [ 746.550040] env[62736]: DEBUG nova.policy [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce8f935dca594cfa87b680e92f1c492c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a34141edbd4482d9f709ad211179a90', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 746.789313] env[62736]: INFO nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Deleted allocations for instance 459055aa-ae56-480f-b2c6-e45cc05bb50c [ 746.795951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 495ebc353c444dd98f4905f4a67ef1a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.811362] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 495ebc353c444dd98f4905f4a67ef1a1 [ 746.845752] env[62736]: INFO nova.compute.manager [-] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Took 1.04 seconds to deallocate network for instance. [ 746.850550] env[62736]: DEBUG nova.compute.claims [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 746.850771] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.881147] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df84abd1-abe2-4089-9afe-5bb7d40c47e5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.892440] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b8b1b82-9265-4cc7-b710-26cb853ca327 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.925648] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e466a70-b9e8-4989-86ec-cd60d1859691 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.934293] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dae5b4f6-9026-4fbd-8db4-7de2303b5fdc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 746.948356] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 746.948866] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg c6a1a81e933d4077b7abe5f1569c7300 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 746.950149] env[62736]: DEBUG oslo_concurrency.lockutils [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] Releasing lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 746.950347] env[62736]: DEBUG nova.compute.manager [req-11f97fa0-36cf-4580-8318-081e30819673 req-8ea2c9b5-fd8c-435a-b604-bf0b96469d3c service nova] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Received event network-vif-deleted-cb3537e6-adcb-4b16-9fc1-bb454d4e1c56 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 746.956095] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.956379] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.961600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6a1a81e933d4077b7abe5f1569c7300 [ 746.977008] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 746.979693] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg c0441ff1b7394bf7886b8cd21040ab83 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.018026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0441ff1b7394bf7886b8cd21040ab83 [ 747.035335] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Successfully created port: fda4aa7d-b176-4a8a-8311-135c11345fee {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 747.297823] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "459055aa-ae56-480f-b2c6-e45cc05bb50c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 145.763s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.298446] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 9fc2982b32624190b44ea8c919c1257a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.310979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fc2982b32624190b44ea8c919c1257a [ 747.452621] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 747.455044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg f017d73cea1e4fd4acec54902f702f94 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.468334] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f017d73cea1e4fd4acec54902f702f94 [ 747.483477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 3f6f5bd2aa744f74a9f5f3bd9cd0bac9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.518222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f6f5bd2aa744f74a9f5f3bd9cd0bac9 [ 747.800371] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 747.802268] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 4122983d27dc4bb88adb863cd3e5582e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.838214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4122983d27dc4bb88adb863cd3e5582e [ 747.944916] env[62736]: DEBUG nova.compute.manager [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Received event network-changed-fda4aa7d-b176-4a8a-8311-135c11345fee {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 747.945112] env[62736]: DEBUG nova.compute.manager [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Refreshing instance network info cache due to event network-changed-fda4aa7d-b176-4a8a-8311-135c11345fee. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 747.945325] env[62736]: DEBUG oslo_concurrency.lockutils [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] Acquiring lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 747.945468] env[62736]: DEBUG oslo_concurrency.lockutils [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] Acquired lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 747.945636] env[62736]: DEBUG nova.network.neutron [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Refreshing network info cache for port fda4aa7d-b176-4a8a-8311-135c11345fee {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 747.946061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] Expecting reply to msg 425fc3f755154b2ba6f1b2b7847fd359 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.953038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 425fc3f755154b2ba6f1b2b7847fd359 [ 747.957883] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.996s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.958462] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Traceback (most recent call last): [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.driver.spawn(context, instance, image_meta, [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] vm_ref = self.build_virtual_machine(instance, [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] vif_infos = vmwarevif.get_vif_info(self._session, [ 747.958462] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] for vif in network_info: [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return self._sync_wrapper(fn, *args, **kwargs) [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.wait() [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self[:] = self._gt.wait() [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return self._exit_event.wait() [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] current.throw(*self._exc) [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 747.958854] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] result = function(*args, **kwargs) [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] return func(*args, **kwargs) [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise e [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] nwinfo = self.network_api.allocate_for_instance( [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] created_port_ids = self._update_ports_for_instance( [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] with excutils.save_and_reraise_exception(): [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] self.force_reraise() [ 747.959291] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise self.value [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] updated_port = self._update_port( [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] _ensure_no_port_binding_failure(port) [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] raise exception.PortBindingFailed(port_id=port['id']) [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] nova.exception.PortBindingFailed: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. [ 747.959735] env[62736]: ERROR nova.compute.manager [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] [ 747.959735] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 747.960634] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Build of instance 5041b2c1-e7f1-4039-b7c4-1f677918025a was re-scheduled: Binding failed for port 90717c60-aad7-4e97-b134-e7d76933ad99, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 747.961282] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 747.961509] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 747.961664] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 747.961984] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 747.962209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 71832d99da6440b199c84ee15e4bc8f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.963026] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.708s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 747.965032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 8f97ae8be14948b294989610dcb1d646 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 747.971360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 71832d99da6440b199c84ee15e4bc8f4 [ 747.986574] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 748.004607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f97ae8be14948b294989610dcb1d646 [ 748.012846] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.013097] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.013251] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.013431] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.013573] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.013713] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.013912] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.014065] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.014221] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.014377] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.014541] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.015596] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-708a8e72-8eab-4a26-af87-222488758529 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.025884] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27735e8e-1388-4d73-892d-da3d1bfa43b4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.175753] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 748.175753] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.175753] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.175753] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.175753] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.175753] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.175753] env[62736]: ERROR nova.compute.manager raise self.value [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.175753] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 748.175753] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.175753] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 748.176278] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.176278] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 748.176278] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 748.176278] env[62736]: ERROR nova.compute.manager [ 748.176278] env[62736]: Traceback (most recent call last): [ 748.176278] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 748.176278] env[62736]: listener.cb(fileno) [ 748.176278] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.176278] env[62736]: result = function(*args, **kwargs) [ 748.176278] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.176278] env[62736]: return func(*args, **kwargs) [ 748.176278] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.176278] env[62736]: raise e [ 748.176278] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.176278] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 748.176278] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.176278] env[62736]: created_port_ids = self._update_ports_for_instance( [ 748.176278] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.176278] env[62736]: with excutils.save_and_reraise_exception(): [ 748.176278] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.176278] env[62736]: self.force_reraise() [ 748.176278] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.176278] env[62736]: raise self.value [ 748.176278] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.176278] env[62736]: updated_port = self._update_port( [ 748.176278] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.176278] env[62736]: _ensure_no_port_binding_failure(port) [ 748.176278] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.176278] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 748.177179] env[62736]: nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 748.177179] env[62736]: Removing descriptor: 17 [ 748.177179] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Traceback (most recent call last): [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] yield resources [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.driver.spawn(context, instance, image_meta, [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 748.177179] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] vm_ref = self.build_virtual_machine(instance, [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] vif_infos = vmwarevif.get_vif_info(self._session, [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] for vif in network_info: [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self._sync_wrapper(fn, *args, **kwargs) [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.wait() [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self[:] = self._gt.wait() [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self._exit_event.wait() [ 748.177630] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] result = hub.switch() [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self.greenlet.switch() [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] result = function(*args, **kwargs) [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return func(*args, **kwargs) [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise e [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] nwinfo = self.network_api.allocate_for_instance( [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 748.178173] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] created_port_ids = self._update_ports_for_instance( [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] with excutils.save_and_reraise_exception(): [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.force_reraise() [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise self.value [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] updated_port = self._update_port( [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] _ensure_no_port_binding_failure(port) [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 748.178566] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise exception.PortBindingFailed(port_id=port['id']) [ 748.178933] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 748.178933] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] [ 748.178933] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Terminating instance [ 748.179571] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 748.323621] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.464037] env[62736]: DEBUG nova.network.neutron [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 748.484491] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 748.569245] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.569792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 5a7c4f36e9584e39a862d9d5c734a7c2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 748.578490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a7c4f36e9584e39a862d9d5c734a7c2 [ 748.591468] env[62736]: DEBUG nova.network.neutron [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.591973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] Expecting reply to msg 711944c102704553806507e5cfaddd39 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 748.599919] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 711944c102704553806507e5cfaddd39 [ 748.786683] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd1d2843-f38b-46f8-bfce-adf5f703a103 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.796828] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22d5c19a-d0ac-46b7-b0d4-f81e9464cc5d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.827989] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de5f2273-9a6f-40f4-9a4b-81012aab7878 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.835114] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62248519-d9d8-4852-9539-f39a743f64e1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.848340] env[62736]: DEBUG nova.compute.provider_tree [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 748.849486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg bf6167e4877344c0aae57c32df6c4655 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 748.856317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf6167e4877344c0aae57c32df6c4655 [ 749.072711] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-5041b2c1-e7f1-4039-b7c4-1f677918025a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 749.073063] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 749.073294] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 749.073506] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 749.089729] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 749.091899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg d823258aad064f6d8e634293246941b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.094600] env[62736]: DEBUG oslo_concurrency.lockutils [req-579fac0e-8d79-4bcd-968f-b8b3ebdddd7f req-1c16bfee-3954-4b43-a333-fd4312fd3e00 service nova] Releasing lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 749.094687] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 749.094867] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 749.095303] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 83a6589f30814f40a852f5ed754d58e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.098645] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d823258aad064f6d8e634293246941b7 [ 749.103869] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83a6589f30814f40a852f5ed754d58e5 [ 749.351854] env[62736]: DEBUG nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 749.354383] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg ff8924fea3d04ae2b946ad41ae4ae34d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.368575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff8924fea3d04ae2b946ad41ae4ae34d [ 749.593656] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.594198] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 2a67efa7c751476fa3f08f530a83e0fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.603306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a67efa7c751476fa3f08f530a83e0fa [ 749.624954] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 749.703541] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 749.704078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg c4c30ea3093f4ac9bcd17019b2a963f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.711890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4c30ea3093f4ac9bcd17019b2a963f0 [ 749.856783] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.894s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.857537] env[62736]: ERROR nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Traceback (most recent call last): [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.driver.spawn(context, instance, image_meta, [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] vm_ref = self.build_virtual_machine(instance, [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] vif_infos = vmwarevif.get_vif_info(self._session, [ 749.857537] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] for vif in network_info: [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return self._sync_wrapper(fn, *args, **kwargs) [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.wait() [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self[:] = self._gt.wait() [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return self._exit_event.wait() [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] current.throw(*self._exc) [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 749.857869] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] result = function(*args, **kwargs) [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] return func(*args, **kwargs) [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise e [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] nwinfo = self.network_api.allocate_for_instance( [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] created_port_ids = self._update_ports_for_instance( [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] with excutils.save_and_reraise_exception(): [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] self.force_reraise() [ 749.858321] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise self.value [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] updated_port = self._update_port( [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] _ensure_no_port_binding_failure(port) [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] raise exception.PortBindingFailed(port_id=port['id']) [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] nova.exception.PortBindingFailed: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. [ 749.858739] env[62736]: ERROR nova.compute.manager [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] [ 749.858739] env[62736]: DEBUG nova.compute.utils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 749.859812] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.082s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.861326] env[62736]: INFO nova.compute.claims [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 749.862817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg c063d391a2ef4701b26663ab5b8933ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.863970] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Build of instance 896c2790-9107-4d1c-811b-07a87582c0a1 was re-scheduled: Binding failed for port 5b2db879-2f10-4314-a5f0-c1f237227a8e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 749.864472] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 749.864630] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquiring lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 749.864773] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Acquired lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 749.864928] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 749.865271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 170a1d7cc2074964aac5dd6c1bde16b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 749.871413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 170a1d7cc2074964aac5dd6c1bde16b2 [ 749.895529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c063d391a2ef4701b26663ab5b8933ff [ 749.983384] env[62736]: DEBUG nova.compute.manager [req-0d26d80a-b9f8-4568-a469-a814b3baabbc req-0082e181-26b9-4d42-8eaf-b5a106a36db4 service nova] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Received event network-vif-deleted-fda4aa7d-b176-4a8a-8311-135c11345fee {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 750.096734] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 5041b2c1-e7f1-4039-b7c4-1f677918025a] Took 1.02 seconds to deallocate network for instance. [ 750.098841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 876b3ff9404b4396a94d34cc29bed27f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.131520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 876b3ff9404b4396a94d34cc29bed27f [ 750.206014] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 750.206331] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 750.206527] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 750.206826] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bd18ab79-9ee8-4c44-8513-f90c6030228c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.216947] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7382f988-76b9-4fb3-9147-8d2956ab458f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 750.237698] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d633de23-23ab-4297-b2d6-2702906b31a9 could not be found. [ 750.237932] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 750.238105] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 750.238341] env[62736]: DEBUG oslo.service.loopingcall [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 750.238551] env[62736]: DEBUG nova.compute.manager [-] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 750.238656] env[62736]: DEBUG nova.network.neutron [-] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 750.258849] env[62736]: DEBUG nova.network.neutron [-] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.258849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 761aca6952444050bd72ccf8f630dbc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.264783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 761aca6952444050bd72ccf8f630dbc8 [ 750.367821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg cd08236f792b4dd0b124b9ae35d93c39 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.375718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd08236f792b4dd0b124b9ae35d93c39 [ 750.383638] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.472236] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.472760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 59f734da3d254e3b9927bb3d27579541 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.480290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59f734da3d254e3b9927bb3d27579541 [ 750.603072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 969d8073026d4aa9a3658f922625bd7a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.635033] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 969d8073026d4aa9a3658f922625bd7a [ 750.760593] env[62736]: DEBUG nova.network.neutron [-] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 750.761059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 935206cf0bb24abc9360a2eac805c09d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.775376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 935206cf0bb24abc9360a2eac805c09d [ 750.974493] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Releasing lock "refresh_cache-896c2790-9107-4d1c-811b-07a87582c0a1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 750.974727] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 750.974902] env[62736]: DEBUG nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 750.975066] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 750.991254] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.991838] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 65aa48fea211409698f665ce97aa6d69 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 750.999426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65aa48fea211409698f665ce97aa6d69 [ 751.121385] env[62736]: INFO nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Deleted allocations for instance 5041b2c1-e7f1-4039-b7c4-1f677918025a [ 751.127032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 679cfeaf17e04df385045ed98142e6a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 751.139004] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 679cfeaf17e04df385045ed98142e6a7 [ 751.208320] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a054857e-4f74-4189-9084-d3778016bbbf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.213617] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-153089f4-4198-4883-9e34-822f6b061af4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.243929] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae952164-300d-471a-9abb-82ae43c51a78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.252052] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22fa58eb-82e7-41dd-ad76-3e23b1ddad30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.265146] env[62736]: INFO nova.compute.manager [-] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Took 1.03 seconds to deallocate network for instance. [ 751.265620] env[62736]: DEBUG nova.compute.provider_tree [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 751.266081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg a97f1d6007a54f96a4a09fb66a1e6fa2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 751.274348] env[62736]: DEBUG nova.compute.claims [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 751.274526] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.274987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a97f1d6007a54f96a4a09fb66a1e6fa2 [ 751.494469] env[62736]: DEBUG nova.network.neutron [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 751.495182] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg b161a9729586419fa88867e56515a133 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 751.504557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b161a9729586419fa88867e56515a133 [ 751.628860] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "5041b2c1-e7f1-4039-b7c4-1f677918025a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 150.028s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 751.629535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg a1fce023220043f5a00a49ec8913b99d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 751.638402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1fce023220043f5a00a49ec8913b99d [ 751.769505] env[62736]: DEBUG nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 751.772578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg eb6d6f2db07e400abb1669449f2e1e7b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 751.786359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb6d6f2db07e400abb1669449f2e1e7b [ 751.997888] env[62736]: INFO nova.compute.manager [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] [instance: 896c2790-9107-4d1c-811b-07a87582c0a1] Took 1.02 seconds to deallocate network for instance. [ 751.999629] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 536b5ef017ae4bd6aabf85308f39eeac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.045523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 536b5ef017ae4bd6aabf85308f39eeac [ 752.131711] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 752.133561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg c616752121554b678592c063bcfefefd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.165094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c616752121554b678592c063bcfefefd [ 752.274964] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.415s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.275593] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 752.277235] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d9f5a8f224324d4c86305ffe31f80f5a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.288349] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.547s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.288349] env[62736]: INFO nova.compute.claims [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 752.288349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 43291136277d4f2aa59cf1bf89743516 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.306277] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9f5a8f224324d4c86305ffe31f80f5a [ 752.330943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43291136277d4f2aa59cf1bf89743516 [ 752.504767] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg ee3a79c44a894d16b518e01ffcabb14a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.534588] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee3a79c44a894d16b518e01ffcabb14a [ 752.650043] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.784085] env[62736]: DEBUG nova.compute.utils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 752.784739] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 15d7794ae17d41028765322a277d6791 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.787154] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 3d9b67339c2741199fbef676d2ae6066 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 752.788067] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 752.788228] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 752.798848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15d7794ae17d41028765322a277d6791 [ 752.805084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d9b67339c2741199fbef676d2ae6066 [ 752.846206] env[62736]: DEBUG nova.policy [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce8f935dca594cfa87b680e92f1c492c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a34141edbd4482d9f709ad211179a90', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 753.027464] env[62736]: INFO nova.scheduler.client.report [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Deleted allocations for instance 896c2790-9107-4d1c-811b-07a87582c0a1 [ 753.033415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Expecting reply to msg 51984d7e459d4f16befaa839def1c361 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 753.043581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51984d7e459d4f16befaa839def1c361 [ 753.215575] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Successfully created port: 9ad2958c-98f6-40ec-8c0b-17728bf04579 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 753.289428] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 753.291203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 7d20c5bfeeb444769318e65bbaaf26b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 753.325510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d20c5bfeeb444769318e65bbaaf26b9 [ 753.535273] env[62736]: DEBUG oslo_concurrency.lockutils [None req-bed3d7b1-e2f6-4237-9326-9c79840a2176 tempest-ListServersNegativeTestJSON-1438981180 tempest-ListServersNegativeTestJSON-1438981180-project-member] Lock "896c2790-9107-4d1c-811b-07a87582c0a1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 151.901s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 753.535839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 000f14443f654dd88f6e5f4532c1a9c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 753.546360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 000f14443f654dd88f6e5f4532c1a9c4 [ 753.608525] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4102fd78-882a-4535-b629-214b044768fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.616479] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea3dd6b1-a14c-4956-8192-3a57ef1e01ef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.646989] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f613e7b-32a3-49f3-b032-8d75456d9b50 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.655023] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a48717e5-c7a6-4cd4-81df-93f63c782cb5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 753.669002] env[62736]: DEBUG nova.compute.provider_tree [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 753.669510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 72fb8088b8aa4513a032f029861e0d0f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 753.677367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72fb8088b8aa4513a032f029861e0d0f [ 753.798520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg eb37abb7712b4f479088289be3c08935 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 753.831562] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb37abb7712b4f479088289be3c08935 [ 754.038110] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 754.040123] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 60165f67bad248f3bd102e912917aba6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.072825] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60165f67bad248f3bd102e912917aba6 [ 754.117739] env[62736]: DEBUG nova.compute.manager [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Received event network-changed-9ad2958c-98f6-40ec-8c0b-17728bf04579 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 754.118002] env[62736]: DEBUG nova.compute.manager [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Refreshing instance network info cache due to event network-changed-9ad2958c-98f6-40ec-8c0b-17728bf04579. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 754.118267] env[62736]: DEBUG oslo_concurrency.lockutils [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] Acquiring lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 754.118451] env[62736]: DEBUG oslo_concurrency.lockutils [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] Acquired lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 754.118639] env[62736]: DEBUG nova.network.neutron [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Refreshing network info cache for port 9ad2958c-98f6-40ec-8c0b-17728bf04579 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 754.119188] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] Expecting reply to msg 172c52608d4a4f26b2e62d6738841a9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.126395] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 172c52608d4a4f26b2e62d6738841a9f [ 754.177006] env[62736]: DEBUG nova.scheduler.client.report [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 754.179554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 079f91234c794f869e452c55a98cc19d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.191539] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 079f91234c794f869e452c55a98cc19d [ 754.301966] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 754.327911] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 754.328230] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 754.328426] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 754.328645] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 754.328825] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 754.329001] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 754.329240] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 754.329520] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 754.329764] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 754.329972] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 754.330181] env[62736]: DEBUG nova.virt.hardware [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 754.331053] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87c7deb2-f915-4752-b420-2c98f78b5bf2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.339403] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e1a2a51-62ff-47fd-8006-cddf512239c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 754.516804] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 754.516804] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 754.516804] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 754.516804] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 754.516804] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 754.516804] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 754.516804] env[62736]: ERROR nova.compute.manager raise self.value [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 754.516804] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 754.516804] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 754.516804] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 754.517292] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 754.517292] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 754.517292] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 754.517292] env[62736]: ERROR nova.compute.manager [ 754.517292] env[62736]: Traceback (most recent call last): [ 754.517292] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 754.517292] env[62736]: listener.cb(fileno) [ 754.517292] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 754.517292] env[62736]: result = function(*args, **kwargs) [ 754.517292] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 754.517292] env[62736]: return func(*args, **kwargs) [ 754.517292] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 754.517292] env[62736]: raise e [ 754.517292] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 754.517292] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 754.517292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 754.517292] env[62736]: created_port_ids = self._update_ports_for_instance( [ 754.517292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 754.517292] env[62736]: with excutils.save_and_reraise_exception(): [ 754.517292] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 754.517292] env[62736]: self.force_reraise() [ 754.517292] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 754.517292] env[62736]: raise self.value [ 754.517292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 754.517292] env[62736]: updated_port = self._update_port( [ 754.517292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 754.517292] env[62736]: _ensure_no_port_binding_failure(port) [ 754.517292] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 754.517292] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 754.518107] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 754.518107] env[62736]: Removing descriptor: 17 [ 754.518107] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Traceback (most recent call last): [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] yield resources [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.driver.spawn(context, instance, image_meta, [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 754.518107] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] vm_ref = self.build_virtual_machine(instance, [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] vif_infos = vmwarevif.get_vif_info(self._session, [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] for vif in network_info: [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self._sync_wrapper(fn, *args, **kwargs) [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.wait() [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self[:] = self._gt.wait() [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self._exit_event.wait() [ 754.518445] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] result = hub.switch() [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self.greenlet.switch() [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] result = function(*args, **kwargs) [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return func(*args, **kwargs) [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise e [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] nwinfo = self.network_api.allocate_for_instance( [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 754.518819] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] created_port_ids = self._update_ports_for_instance( [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] with excutils.save_and_reraise_exception(): [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.force_reraise() [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise self.value [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] updated_port = self._update_port( [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] _ensure_no_port_binding_failure(port) [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 754.519181] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise exception.PortBindingFailed(port_id=port['id']) [ 754.519737] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 754.519737] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] [ 754.519737] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Terminating instance [ 754.520146] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 754.561702] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.634065] env[62736]: DEBUG nova.network.neutron [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 754.683945] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.404s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.683945] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 754.684871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 7e4c9b29b4924011a742cbc089e9a82a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.687925] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.864s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.689284] env[62736]: INFO nova.compute.claims [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 754.691809] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg f9826e30e14b448584b616cdd17359f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.720327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e4c9b29b4924011a742cbc089e9a82a [ 754.724419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f9826e30e14b448584b616cdd17359f1 [ 754.741960] env[62736]: DEBUG nova.network.neutron [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 754.742481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] Expecting reply to msg a6897159dbc04293928d5b97995a0d2a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 754.750203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6897159dbc04293928d5b97995a0d2a [ 755.195168] env[62736]: DEBUG nova.compute.utils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 755.195828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 1dd4313564d2416898804d9c5c78913a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 755.197972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 98063e9a24c1415dbd3c6213ee5f8aaf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 755.198742] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 755.206274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98063e9a24c1415dbd3c6213ee5f8aaf [ 755.207884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1dd4313564d2416898804d9c5c78913a [ 755.245240] env[62736]: DEBUG oslo_concurrency.lockutils [req-e6bc5acf-94f8-4985-bcd7-337b7fe65af1 req-5c3d3a64-3df1-4b2c-a01a-f8eebbba8c8f service nova] Releasing lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 755.245616] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 755.245824] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 755.246220] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 299b57383a2949a18d729075b8685302 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 755.252261] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 299b57383a2949a18d729075b8685302 [ 755.703473] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 755.705057] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 1161b514aa22474faf6bdd200d2096a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 755.739053] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1161b514aa22474faf6bdd200d2096a7 [ 755.787233] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 755.986427] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 755.986983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg bf4c091b3c0d40d0aadd965b169aa989 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 755.995416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf4c091b3c0d40d0aadd965b169aa989 [ 756.026511] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f00846e-eec9-46f5-be16-558212d6d3ea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.038279] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7d22664-8478-4b5a-af0f-bcb0e5d6a0b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.068669] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-987106b0-ec9e-4a38-ae7e-fefefaca1f56 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.081040] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300d03a7-ef78-4ca9-a4e5-f5ea079f41ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.095303] env[62736]: DEBUG nova.compute.provider_tree [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 756.095945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 37d95c4bc16b4c02834d015b0566be3f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 756.110421] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37d95c4bc16b4c02834d015b0566be3f [ 756.146798] env[62736]: DEBUG nova.compute.manager [req-983237cb-41ab-4ba4-b179-a0db5fc828fd req-c3fb7670-5dd1-4962-acb1-ee896ed7b952 service nova] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Received event network-vif-deleted-9ad2958c-98f6-40ec-8c0b-17728bf04579 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 756.209620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg acc4bf41530c424a8a9db11584f47b4b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 756.242458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acc4bf41530c424a8a9db11584f47b4b [ 756.489369] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 756.489830] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 756.490013] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 756.490331] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-107742f4-b55f-4385-9107-e50f67185dde {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.498987] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e89c99f9-6404-4e46-9f3a-bba9d1521c84 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.519649] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0 could not be found. [ 756.519772] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 756.519941] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Took 0.03 seconds to destroy the instance on the hypervisor. [ 756.520185] env[62736]: DEBUG oslo.service.loopingcall [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 756.520388] env[62736]: DEBUG nova.compute.manager [-] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 756.520486] env[62736]: DEBUG nova.network.neutron [-] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 756.533257] env[62736]: DEBUG nova.network.neutron [-] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.533777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 133859af05994c79b69325420cdc0ac8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 756.540665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 133859af05994c79b69325420cdc0ac8 [ 756.599496] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 756.602146] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 95c86866d8a24aac801058ca1f8d129a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 756.617296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95c86866d8a24aac801058ca1f8d129a [ 756.713980] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 756.756266] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 756.756266] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 756.756266] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 756.756266] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 756.756712] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 756.756712] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 756.756712] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 756.756712] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 756.756712] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 756.756892] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 756.756892] env[62736]: DEBUG nova.virt.hardware [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 756.756892] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4343df2c-33cc-46fa-8116-df51fb54653a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.771624] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc193282-eb59-45ad-9d03-9f0de9d9d60a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.794274] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 756.800297] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Creating folder: Project (82ebdd23e4964eedba2d5a377c62c702). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 756.800777] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9e4a06e8-0401-4555-9d1c-caa379d09aac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.811685] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Created folder: Project (82ebdd23e4964eedba2d5a377c62c702) in parent group-v107913. [ 756.812163] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Creating folder: Instances. Parent ref: group-v107926. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 756.812514] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b2c0098f-c0c4-4285-8cf2-27f63e41c8c9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.821076] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Created folder: Instances in parent group-v107926. [ 756.821472] env[62736]: DEBUG oslo.service.loopingcall [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 756.821833] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 756.822160] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-6b5d91cb-6d7d-4a85-94b8-f173ad9823f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 756.837999] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 756.837999] env[62736]: value = "task-397716" [ 756.837999] env[62736]: _type = "Task" [ 756.837999] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 756.847006] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397716, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 757.035631] env[62736]: DEBUG nova.network.neutron [-] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.036153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 724c341dee1d4795b03dd3c6ac5488fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 757.045269] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 724c341dee1d4795b03dd3c6ac5488fc [ 757.105419] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.417s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.105973] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 757.107739] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg f7b2f95cd09649a5b5481e8ac3bdc0d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 757.108644] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.420s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.110504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 44f047a4b01e43c9805ab28192fa9ecd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 757.141920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7b2f95cd09649a5b5481e8ac3bdc0d9 [ 757.144022] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44f047a4b01e43c9805ab28192fa9ecd [ 757.349107] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397716, 'name': CreateVM_Task, 'duration_secs': 0.305139} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 757.349284] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 757.349787] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 757.349948] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 757.350278] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 757.350537] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-19022b74-89cd-4eda-b65b-d42d38fbbaf3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.354878] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 757.354878] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5231c77d-8f48-3448-02ea-17fada0ddb9d" [ 757.354878] env[62736]: _type = "Task" [ 757.354878] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 757.362133] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5231c77d-8f48-3448-02ea-17fada0ddb9d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 757.538405] env[62736]: INFO nova.compute.manager [-] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Took 1.02 seconds to deallocate network for instance. [ 757.540896] env[62736]: DEBUG nova.compute.claims [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 757.541064] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.613544] env[62736]: DEBUG nova.compute.utils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 757.614207] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 3584ff2aa26f4116bc752377a41a153f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 757.618137] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 757.618305] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 757.627976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3584ff2aa26f4116bc752377a41a153f [ 757.681230] env[62736]: DEBUG nova.policy [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '613209c7c103464ea473f32bcf0116b1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c343033abe74ef28f09e614ffad9254', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 757.865441] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5231c77d-8f48-3448-02ea-17fada0ddb9d, 'name': SearchDatastore_Task, 'duration_secs': 0.012657} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 757.865756] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 757.866023] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 757.866263] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 757.866406] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 757.866580] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 757.866837] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-91eda189-3f04-4d29-8f73-88ef0ea11d21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.874983] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 757.875173] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 757.875902] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b9caffd3-378b-4a7b-ab87-e2b9d607b252 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.883356] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 757.883356] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]525e0c42-e1c2-9d58-a650-6e184960c2b8" [ 757.883356] env[62736]: _type = "Task" [ 757.883356] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 757.901015] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525e0c42-e1c2-9d58-a650-6e184960c2b8, 'name': SearchDatastore_Task, 'duration_secs': 0.008639} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 757.907303] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-869a7438-dd01-4e38-9653-64d279893bdc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.909633] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 757.909633] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52c9ba9f-f5a7-27e8-9107-706dbbcd5864" [ 757.909633] env[62736]: _type = "Task" [ 757.909633] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 757.918801] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52c9ba9f-f5a7-27e8-9107-706dbbcd5864, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 757.969552] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9afa21b2-0471-497a-9115-1166eed19342 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 757.976474] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a075cb35-ee9b-4000-a779-1db0cc2ff471 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.010237] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89c35c4d-020d-4a84-b657-66caf4e56af1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.017860] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-334e06a9-47f1-48f4-879e-0a34b0454066 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.040078] env[62736]: DEBUG nova.compute.provider_tree [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 758.040078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg ee5ef448ff6847d2b11666e7877a5c9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 758.046306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ee5ef448ff6847d2b11666e7877a5c9f [ 758.104848] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Successfully created port: 21f35458-40bc-40fc-8381-9814ec25b9cb {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 758.119137] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 758.121027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 9b2e9932da2549b38a5e6bcb591fb447 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 758.150420] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b2e9932da2549b38a5e6bcb591fb447 [ 758.424891] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52c9ba9f-f5a7-27e8-9107-706dbbcd5864, 'name': SearchDatastore_Task, 'duration_secs': 0.008818} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 758.424891] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 758.424891] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 758.424891] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Successfully created port: 41a9c7ab-2ce6-4d42-a6d4-9997c021959e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 758.425242] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1f59b623-0a46-4fa2-a1eb-eeb299be2aa4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.429653] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 758.429653] env[62736]: value = "task-397717" [ 758.429653] env[62736]: _type = "Task" [ 758.429653] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 758.437797] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397717, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 758.542170] env[62736]: DEBUG nova.scheduler.client.report [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 758.544966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 5eeb3b1a9141475990d823e4b79446fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 758.557156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5eeb3b1a9141475990d823e4b79446fe [ 758.636032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg a4bf688ffc0f4cee8482d7e316c0e6be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 758.677615] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4bf688ffc0f4cee8482d7e316c0e6be [ 758.868833] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Successfully created port: b076cf9c-01b0-441f-9392-404a633e5ed5 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 758.942531] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397717, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.454146} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 758.942531] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 758.942531] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 758.942531] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-c128adbb-6a28-4376-b6b5-a149003fd616 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 758.947008] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 758.947008] env[62736]: value = "task-397718" [ 758.947008] env[62736]: _type = "Task" [ 758.947008] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 758.954975] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397718, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 759.048216] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.939s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 759.049037] env[62736]: ERROR nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Traceback (most recent call last): [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.driver.spawn(context, instance, image_meta, [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self._vmops.spawn(context, instance, image_meta, injected_files, [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] vm_ref = self.build_virtual_machine(instance, [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] vif_infos = vmwarevif.get_vif_info(self._session, [ 759.049037] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] for vif in network_info: [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self._sync_wrapper(fn, *args, **kwargs) [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.wait() [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self[:] = self._gt.wait() [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self._exit_event.wait() [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] result = hub.switch() [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 759.049485] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return self.greenlet.switch() [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] result = function(*args, **kwargs) [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] return func(*args, **kwargs) [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise e [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] nwinfo = self.network_api.allocate_for_instance( [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] created_port_ids = self._update_ports_for_instance( [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] with excutils.save_and_reraise_exception(): [ 759.049917] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] self.force_reraise() [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise self.value [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] updated_port = self._update_port( [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] _ensure_no_port_binding_failure(port) [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] raise exception.PortBindingFailed(port_id=port['id']) [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] nova.exception.PortBindingFailed: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. [ 759.050363] env[62736]: ERROR nova.compute.manager [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] [ 759.051793] env[62736]: DEBUG nova.compute.utils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 759.053118] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 15.240s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 759.053423] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 759.053700] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 759.054077] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.922s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 759.056247] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 9f74f425109745cda3039d115039c165 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 759.057763] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Build of instance 1174dcea-6ba6-4a2b-9eb4-85e662691c30 was re-scheduled: Binding failed for port 28401104-aeb3-4305-8c00-72354272efa2, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 759.058360] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 759.058704] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquiring lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 759.058961] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Acquired lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 759.059231] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 759.059730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 69acaa25a6904120be628651940a7d04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 759.065067] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d453eccc-928b-46bb-9eb0-7b688e63b987 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.072210] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69acaa25a6904120be628651940a7d04 [ 759.076850] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32d77950-761b-44f0-a2a6-a9d7681bb70f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.093609] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee202426-f773-4a7f-b03a-4b587f2f1928 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.097557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f74f425109745cda3039d115039c165 [ 759.100870] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb85e2ff-f91a-47ef-949f-944a5e520b7d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.134241] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181568MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 759.134560] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 759.151392] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 759.181126] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 759.181622] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 759.182040] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 759.182383] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 759.182651] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 759.182934] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 759.183311] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 759.183601] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 759.183899] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 759.184207] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 759.184510] env[62736]: DEBUG nova.virt.hardware [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 759.185528] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84e80c28-d731-41ba-b3a6-17e7519dd7f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.195422] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e526fb64-3083-4e93-aed3-f6a887d5f4eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.457489] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397718, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.062204} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 759.457897] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 759.458742] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5db878b6-46e2-4e5e-a32d-73e786112613 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.478358] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfiguring VM instance instance-00000032 to attach disk [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 759.478774] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-46cfd7b0-309c-4bde-8b49-fe8d464ab260 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.498116] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 759.498116] env[62736]: value = "task-397719" [ 759.498116] env[62736]: _type = "Task" [ 759.498116] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 759.507392] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397719, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 759.596613] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 759.673961] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.674938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 8bb43ca7427e468a8cdae790353d9470 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 759.686963] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8bb43ca7427e468a8cdae790353d9470 [ 759.747392] env[62736]: DEBUG nova.compute.manager [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Received event network-changed-21f35458-40bc-40fc-8381-9814ec25b9cb {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 759.747814] env[62736]: DEBUG nova.compute.manager [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Refreshing instance network info cache due to event network-changed-21f35458-40bc-40fc-8381-9814ec25b9cb. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 759.748164] env[62736]: DEBUG oslo_concurrency.lockutils [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] Acquiring lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 759.748625] env[62736]: DEBUG oslo_concurrency.lockutils [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] Acquired lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 759.749023] env[62736]: DEBUG nova.network.neutron [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Refreshing network info cache for port 21f35458-40bc-40fc-8381-9814ec25b9cb {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 759.749565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] Expecting reply to msg 98e24177c77c41d685d34e32053eb00b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 759.756209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98e24177c77c41d685d34e32053eb00b [ 759.921455] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97952aaf-e4ff-4869-b00b-a0feab8e85eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.928461] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-869acc57-5d2c-4fab-b4f5-9ca40cbb6181 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.957954] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-635d4d2f-9a38-454d-bd54-c2b9fb6990ce {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.965049] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f3ac168-2ed6-4a48-af2e-0889b52faeef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 759.969428] env[62736]: ERROR nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 759.969428] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 759.969428] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 759.969428] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 759.969428] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 759.969428] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 759.969428] env[62736]: ERROR nova.compute.manager raise self.value [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 759.969428] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 759.969428] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 759.969428] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 759.969937] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 759.969937] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 759.969937] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 759.969937] env[62736]: ERROR nova.compute.manager [ 759.969937] env[62736]: Traceback (most recent call last): [ 759.969937] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 759.969937] env[62736]: listener.cb(fileno) [ 759.969937] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 759.969937] env[62736]: result = function(*args, **kwargs) [ 759.969937] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 759.969937] env[62736]: return func(*args, **kwargs) [ 759.969937] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 759.969937] env[62736]: raise e [ 759.969937] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 759.969937] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 759.969937] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 759.969937] env[62736]: created_port_ids = self._update_ports_for_instance( [ 759.969937] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 759.969937] env[62736]: with excutils.save_and_reraise_exception(): [ 759.969937] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 759.969937] env[62736]: self.force_reraise() [ 759.969937] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 759.969937] env[62736]: raise self.value [ 759.969937] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 759.969937] env[62736]: updated_port = self._update_port( [ 759.969937] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 759.969937] env[62736]: _ensure_no_port_binding_failure(port) [ 759.969937] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 759.969937] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 759.970952] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 759.970952] env[62736]: Removing descriptor: 17 [ 759.970952] env[62736]: ERROR nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Traceback (most recent call last): [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] yield resources [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.driver.spawn(context, instance, image_meta, [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self._vmops.spawn(context, instance, image_meta, injected_files, [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 759.970952] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] vm_ref = self.build_virtual_machine(instance, [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] vif_infos = vmwarevif.get_vif_info(self._session, [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] for vif in network_info: [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self._sync_wrapper(fn, *args, **kwargs) [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.wait() [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self[:] = self._gt.wait() [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self._exit_event.wait() [ 759.971475] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] result = hub.switch() [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self.greenlet.switch() [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] result = function(*args, **kwargs) [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return func(*args, **kwargs) [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise e [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] nwinfo = self.network_api.allocate_for_instance( [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 759.971929] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] created_port_ids = self._update_ports_for_instance( [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] with excutils.save_and_reraise_exception(): [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.force_reraise() [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise self.value [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] updated_port = self._update_port( [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] _ensure_no_port_binding_failure(port) [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 759.972330] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise exception.PortBindingFailed(port_id=port['id']) [ 759.972714] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 759.972714] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] [ 759.972714] env[62736]: INFO nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Terminating instance [ 759.972812] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 759.982076] env[62736]: DEBUG nova.compute.provider_tree [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 759.982306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 5f287018e1984ab29ab07b05dc40ebef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 759.989344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f287018e1984ab29ab07b05dc40ebef [ 760.006871] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397719, 'name': ReconfigVM_Task, 'duration_secs': 0.29501} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 760.007139] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfigured VM instance instance-00000032 to attach disk [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 760.007738] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-781782d4-732f-46cf-9c08-cd5032d4c866 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.013428] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 760.013428] env[62736]: value = "task-397720" [ 760.013428] env[62736]: _type = "Task" [ 760.013428] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 760.022393] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397720, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 760.177844] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Releasing lock "refresh_cache-1174dcea-6ba6-4a2b-9eb4-85e662691c30" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 760.178094] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 760.178278] env[62736]: DEBUG nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 760.178487] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 760.193446] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.194003] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 7955fbaac2e3451d905e261dc89b2789 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.200862] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7955fbaac2e3451d905e261dc89b2789 [ 760.302304] env[62736]: DEBUG nova.network.neutron [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.393975] env[62736]: DEBUG nova.network.neutron [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.394595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] Expecting reply to msg 515f131c3cc94b2197e3c58346c50714 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.404105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 515f131c3cc94b2197e3c58346c50714 [ 760.485559] env[62736]: DEBUG nova.scheduler.client.report [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 760.487673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 781b73401c3142e1b3a26f60d37e50b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.501400] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 781b73401c3142e1b3a26f60d37e50b2 [ 760.522943] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397720, 'name': Rename_Task, 'duration_secs': 0.133815} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 760.523410] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 760.523792] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-c9c61f7c-04b7-4b35-a506-d731e6b1e1c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 760.530625] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 760.530625] env[62736]: value = "task-397721" [ 760.530625] env[62736]: _type = "Task" [ 760.530625] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 760.539283] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397721, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 760.696477] env[62736]: DEBUG nova.network.neutron [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.697407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg dca9b34e743d4a32bd9f4cd33f554eb3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.706940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dca9b34e743d4a32bd9f4cd33f554eb3 [ 760.898558] env[62736]: DEBUG oslo_concurrency.lockutils [req-584819e7-e890-4cb2-b5f5-438f1fe6c0eb req-96503fe8-7402-4883-8dca-2616d0867193 service nova] Releasing lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 760.898558] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquired lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 760.898558] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 760.898558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 3cf753ba089a49b481d23e7da5d4c989 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.905147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cf753ba089a49b481d23e7da5d4c989 [ 760.991483] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.936s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 760.991483] env[62736]: ERROR nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Traceback (most recent call last): [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.driver.spawn(context, instance, image_meta, [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self._vmops.spawn(context, instance, image_meta, injected_files, [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 760.991483] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] vm_ref = self.build_virtual_machine(instance, [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] vif_infos = vmwarevif.get_vif_info(self._session, [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] for vif in network_info: [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self._sync_wrapper(fn, *args, **kwargs) [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.wait() [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self[:] = self._gt.wait() [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self._exit_event.wait() [ 760.992073] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] result = hub.switch() [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return self.greenlet.switch() [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] result = function(*args, **kwargs) [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] return func(*args, **kwargs) [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise e [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] nwinfo = self.network_api.allocate_for_instance( [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 760.992453] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] created_port_ids = self._update_ports_for_instance( [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] with excutils.save_and_reraise_exception(): [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] self.force_reraise() [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise self.value [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] updated_port = self._update_port( [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] _ensure_no_port_binding_failure(port) [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 760.992859] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] raise exception.PortBindingFailed(port_id=port['id']) [ 760.993268] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] nova.exception.PortBindingFailed: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. [ 760.993268] env[62736]: ERROR nova.compute.manager [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] [ 760.993268] env[62736]: DEBUG nova.compute.utils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 760.994140] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Build of instance 1a5a4c2e-74e6-4a62-adda-c3d89deecf05 was re-scheduled: Binding failed for port 701fcdef-8298-4a8d-8739-5a629c9d99e4, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 760.994572] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 760.994810] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquiring lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 760.994940] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Acquired lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 760.995138] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 760.995721] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 408c4441334b4fddaded6efdd2f5f63d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 760.996477] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.146s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 760.998475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 1dd54fd92d304b968d3f9e8a17febae9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.006439] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 408c4441334b4fddaded6efdd2f5f63d [ 761.040989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1dd54fd92d304b968d3f9e8a17febae9 [ 761.041419] env[62736]: DEBUG oslo_vmware.api [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397721, 'name': PowerOnVM_Task, 'duration_secs': 0.47386} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 761.041743] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 761.041847] env[62736]: INFO nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Took 4.33 seconds to spawn the instance on the hypervisor. [ 761.042003] env[62736]: DEBUG nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 761.042845] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09f9d328-3363-4fce-994f-ef4ffab3ce65 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.051978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg e4b3e839435a4dd292e34cd3a546030c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.086008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4b3e839435a4dd292e34cd3a546030c [ 761.199739] env[62736]: INFO nova.compute.manager [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] [instance: 1174dcea-6ba6-4a2b-9eb4-85e662691c30] Took 1.02 seconds to deallocate network for instance. [ 761.201484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 47cecbcf8eb141eabe7f38b9281f2597 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.256189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47cecbcf8eb141eabe7f38b9281f2597 [ 761.452485] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 761.520079] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 761.561008] env[62736]: INFO nova.compute.manager [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Took 24.85 seconds to build instance. [ 761.561571] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 798e1315705348baa3aae90ef4f239bc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.588653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 798e1315705348baa3aae90ef4f239bc [ 761.630768] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.631295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 831cb86bba8c4d3c897731268e572a26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.635517] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.635970] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 9226429b0b604bb5ac60209715d21480 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.644781] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 831cb86bba8c4d3c897731268e572a26 [ 761.645360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9226429b0b604bb5ac60209715d21480 [ 761.709839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 1eb11ad9618441bda4c2f2690a923085 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.745108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1eb11ad9618441bda4c2f2690a923085 [ 761.779883] env[62736]: DEBUG nova.compute.manager [req-d1cd5fc4-f188-4cbe-9c36-b977eae56904 req-2c6cdfc7-77d8-4034-b592-432f8b54f45c service nova] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Received event network-vif-deleted-21f35458-40bc-40fc-8381-9814ec25b9cb {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 761.846629] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08faae90-cb0c-4e3d-b791-4eb02883e6c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.854529] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8b65e18-bfd5-4caf-96f9-7f1844b9812d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.885636] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1495979-39ea-429a-aa67-409d6f8454d1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.894347] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be2752bf-554e-4267-b934-8998db618f4a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 761.909223] env[62736]: DEBUG nova.compute.provider_tree [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 761.909830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 6fa235b6ed104a1cbbda54df8785e8d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 761.916985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fa235b6ed104a1cbbda54df8785e8d7 [ 762.066297] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3ae0a5db-433b-4be8-b161-03d7896dc541 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 128.874s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.067008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg e29975ffa1c54ae3907007cb73d02278 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.082118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e29975ffa1c54ae3907007cb73d02278 [ 762.133706] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Releasing lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 762.134159] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 762.134352] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 762.134651] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cb6fa115-2389-41c9-b711-0a3b48dd544b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.138324] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Releasing lock "refresh_cache-1a5a4c2e-74e6-4a62-adda-c3d89deecf05" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 762.138518] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 762.138689] env[62736]: DEBUG nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 762.138851] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 762.145767] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26d55a10-3d7f-485e-8ea6-2f1cf1d81e78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 762.156189] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 762.156782] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg 7a669cf720ff48f28f7a6144f247f707 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.163907] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a669cf720ff48f28f7a6144f247f707 [ 762.170190] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca could not be found. [ 762.170402] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 762.170585] env[62736]: INFO nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Took 0.04 seconds to destroy the instance on the hypervisor. [ 762.170817] env[62736]: DEBUG oslo.service.loopingcall [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 762.171025] env[62736]: DEBUG nova.compute.manager [-] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 762.171117] env[62736]: DEBUG nova.network.neutron [-] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 762.232547] env[62736]: INFO nova.scheduler.client.report [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Deleted allocations for instance 1174dcea-6ba6-4a2b-9eb4-85e662691c30 [ 762.238756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Expecting reply to msg 999ecc4c67ba42218d590fa379385e7a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.248591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 999ecc4c67ba42218d590fa379385e7a [ 762.257928] env[62736]: DEBUG nova.network.neutron [-] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 762.412842] env[62736]: DEBUG nova.scheduler.client.report [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 762.415342] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg c678cf3dfdb94dbebb34de01d8d6492d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.432609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c678cf3dfdb94dbebb34de01d8d6492d [ 762.478424] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 7e7d570905b64cb1a76cde9af5edcc9e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.491463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e7d570905b64cb1a76cde9af5edcc9e [ 762.569572] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 762.571342] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 788886fd496e472597b914d8b13581eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.606390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2b5d9d03fd084e18b97500915ec3cfc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.607951] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 788886fd496e472597b914d8b13581eb [ 762.613296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b5d9d03fd084e18b97500915ec3cfc6 [ 762.660258] env[62736]: DEBUG nova.network.neutron [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 762.660867] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg c92a2478a8f74cb6a2178bb13d7e10b6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.669166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c92a2478a8f74cb6a2178bb13d7e10b6 [ 762.741158] env[62736]: DEBUG oslo_concurrency.lockutils [None req-fdfdaf1a-a896-4bcc-a512-c865e26faa06 tempest-ServerRescueTestJSONUnderV235-1692950365 tempest-ServerRescueTestJSONUnderV235-1692950365-project-member] Lock "1174dcea-6ba6-4a2b-9eb4-85e662691c30" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 160.807s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.741734] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 2b39ea30de714e85a3957a6822f5c893 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.753623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b39ea30de714e85a3957a6822f5c893 [ 762.919476] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.923s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 762.920168] env[62736]: ERROR nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Traceback (most recent call last): [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.driver.spawn(context, instance, image_meta, [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] vm_ref = self.build_virtual_machine(instance, [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] vif_infos = vmwarevif.get_vif_info(self._session, [ 762.920168] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] for vif in network_info: [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self._sync_wrapper(fn, *args, **kwargs) [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.wait() [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self[:] = self._gt.wait() [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self._exit_event.wait() [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] result = hub.switch() [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 762.920699] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return self.greenlet.switch() [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] result = function(*args, **kwargs) [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] return func(*args, **kwargs) [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise e [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] nwinfo = self.network_api.allocate_for_instance( [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] created_port_ids = self._update_ports_for_instance( [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] with excutils.save_and_reraise_exception(): [ 762.921116] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] self.force_reraise() [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise self.value [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] updated_port = self._update_port( [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] _ensure_no_port_binding_failure(port) [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] raise exception.PortBindingFailed(port_id=port['id']) [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] nova.exception.PortBindingFailed: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. [ 762.921541] env[62736]: ERROR nova.compute.manager [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] [ 762.922031] env[62736]: DEBUG nova.compute.utils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 762.922079] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.599s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 762.923532] env[62736]: INFO nova.compute.claims [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 762.925132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg e4e53bbbab14487493e0e57429da60f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.927195] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Build of instance 427400bc-f265-40cb-8c1c-ccab7bc94b1c was re-scheduled: Binding failed for port cb3537e6-adcb-4b16-9fc1-bb454d4e1c56, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 762.927195] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 762.927195] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquiring lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 762.927195] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Acquired lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 762.927519] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 762.927707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg bc8bf72969ad4c5da9534db24c8404b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 762.934237] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc8bf72969ad4c5da9534db24c8404b2 [ 762.957048] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4e53bbbab14487493e0e57429da60f7 [ 762.982350] env[62736]: INFO nova.compute.manager [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Rebuilding instance [ 763.024462] env[62736]: DEBUG nova.compute.manager [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 763.025416] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69b8d838-9b4c-4b62-a2ca-5c0cde209194 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.033510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 123879d235fa4d609ada6d690d88606a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.062285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 123879d235fa4d609ada6d690d88606a [ 763.097997] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.108732] env[62736]: DEBUG nova.network.neutron [-] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 763.109172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b5fc9bbe286f4ff39f0c57c9a642e60c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.117922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5fc9bbe286f4ff39f0c57c9a642e60c [ 763.162742] env[62736]: INFO nova.compute.manager [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] [instance: 1a5a4c2e-74e6-4a62-adda-c3d89deecf05] Took 1.02 seconds to deallocate network for instance. [ 763.164593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg e1f48419fd6a455fa447b3421c2e5cd5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.199909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1f48419fd6a455fa447b3421c2e5cd5 [ 763.244372] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 763.246253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg c9a33cf72e184cd2b4184de5a5af4ebd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.292526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9a33cf72e184cd2b4184de5a5af4ebd [ 763.430765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 334b6bc722954d48809f6e37fbf6b3b1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.440238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 334b6bc722954d48809f6e37fbf6b3b1 [ 763.446812] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 763.527537] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 763.528118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg b7e34847378748f093f83452b9edbe0d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.537330] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 763.538221] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b7e34847378748f093f83452b9edbe0d [ 763.538599] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-18181247-0a6d-4b70-894e-e37cc06159a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 763.546926] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 763.546926] env[62736]: value = "task-397722" [ 763.546926] env[62736]: _type = "Task" [ 763.546926] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 763.557152] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397722, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 763.611119] env[62736]: INFO nova.compute.manager [-] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Took 1.44 seconds to deallocate network for instance. [ 763.613785] env[62736]: DEBUG nova.compute.claims [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 763.613966] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.669305] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg fbb3645cf06e43689048643637991c3c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 763.703389] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbb3645cf06e43689048643637991c3c [ 763.763240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 764.037062] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Releasing lock "refresh_cache-427400bc-f265-40cb-8c1c-ccab7bc94b1c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 764.037326] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 764.037550] env[62736]: DEBUG nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 764.037728] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 764.057013] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397722, 'name': PowerOffVM_Task, 'duration_secs': 0.214468} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 764.057282] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 764.057503] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 764.058290] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dce2c09b-3782-42d0-9c44-1e15bdc61c9b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.064166] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 764.064709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 7f2998fbd73e47c89906df77fa3a8026 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.070208] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 764.070454] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-564aa099-8133-442a-9113-58ae40655366 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.073427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f2998fbd73e47c89906df77fa3a8026 [ 764.097665] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 764.099060] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 764.099292] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Deleting the datastore file [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 764.099563] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-62ac9e09-2f09-49c6-9a08-67980e733726 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.105219] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 764.105219] env[62736]: value = "task-397724" [ 764.105219] env[62736]: _type = "Task" [ 764.105219] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 764.116139] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397724, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 764.209142] env[62736]: INFO nova.scheduler.client.report [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Deleted allocations for instance 1a5a4c2e-74e6-4a62-adda-c3d89deecf05 [ 764.215322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Expecting reply to msg b44d7fef783145f8a2a7c33fe6beb680 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.233260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b44d7fef783145f8a2a7c33fe6beb680 [ 764.305844] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83646a50-431c-4d44-891b-3beda80fb696 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.313295] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c35ae1d-fb0f-4ddf-8e45-5ec10d87139e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.343508] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9369737-1011-4b4b-957c-c87a77747bab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.350879] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69f2cc42-3a0a-45ff-a60b-bfd2188c7df2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 764.365720] env[62736]: DEBUG nova.compute.provider_tree [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 764.366213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 978227f8863a4d14a51b5b5196606325 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.373050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 978227f8863a4d14a51b5b5196606325 [ 764.568496] env[62736]: DEBUG nova.network.neutron [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 764.568496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 6fec856f6530483a8068178a3af302e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.576267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fec856f6530483a8068178a3af302e5 [ 764.615991] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397724, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.104744} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 764.616272] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 764.616464] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 764.616635] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 764.618223] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 0d8df825701049b5878a0bfd90a1681f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.661456] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d8df825701049b5878a0bfd90a1681f [ 764.720383] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2ec24e8c-1ccd-40cb-a4ca-85767527de4c tempest-ServerActionsTestOtherA-985572683 tempest-ServerActionsTestOtherA-985572683-project-member] Lock "1a5a4c2e-74e6-4a62-adda-c3d89deecf05" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 157.005s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 764.721043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 1f756646c8db4f8ebe1835e7100606b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.731842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f756646c8db4f8ebe1835e7100606b0 [ 764.869656] env[62736]: DEBUG nova.scheduler.client.report [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 764.872103] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg c74942343dd8443cb253b3238ee8c86f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 764.889921] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c74942343dd8443cb253b3238ee8c86f [ 765.070280] env[62736]: INFO nova.compute.manager [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] [instance: 427400bc-f265-40cb-8c1c-ccab7bc94b1c] Took 1.03 seconds to deallocate network for instance. [ 765.073396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg a8abb0d9842648f7999c6b13599aac53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.117590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8abb0d9842648f7999c6b13599aac53 [ 765.122845] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 536bba916a6f42509ed000e976f265ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.157741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 536bba916a6f42509ed000e976f265ee [ 765.224012] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 765.226489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg b76b5166929e4fbe8474bc5225db7995 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.270313] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b76b5166929e4fbe8474bc5225db7995 [ 765.375253] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.453s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 765.375798] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 765.377446] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg d5945a8ea07440e3a34fcece73141321 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.379449] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.105s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 765.381169] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d8cbcf6cf3d840d2af093240fb8f7848 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.414984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8cbcf6cf3d840d2af093240fb8f7848 [ 765.416149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5945a8ea07440e3a34fcece73141321 [ 765.578327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 378728cc12ba41d6bee8c98487a2b9d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.615717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 378728cc12ba41d6bee8c98487a2b9d6 [ 765.652387] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 765.652621] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 765.652775] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 765.652973] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 765.653113] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 765.653727] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 765.653727] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 765.653727] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 765.653941] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 765.653941] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 765.654112] env[62736]: DEBUG nova.virt.hardware [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 765.655299] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe8ef1fe-5a92-4ab2-80d3-1ffccd70ab39 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.671182] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c30400fe-b00e-496d-aafe-64c8e5e56208 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.684809] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 765.690496] env[62736]: DEBUG oslo.service.loopingcall [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 765.691057] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 765.691257] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-c6f09a15-6618-49d8-8735-3d7cf1a89ec8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 765.708009] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 765.708009] env[62736]: value = "task-397725" [ 765.708009] env[62736]: _type = "Task" [ 765.708009] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 765.715631] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397725, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 765.757713] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 765.881159] env[62736]: DEBUG nova.compute.utils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 765.881827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 384e14c2362a4f35b72ac49d10cdf968 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 765.884656] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 765.884656] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 765.894649] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 384e14c2362a4f35b72ac49d10cdf968 [ 765.955511] env[62736]: DEBUG nova.policy [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1a48bd168e18415d8fca989071e048d3', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'eb7d255444a54cc3b2f7ede00ec3bbca', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 766.105343] env[62736]: INFO nova.scheduler.client.report [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Deleted allocations for instance 427400bc-f265-40cb-8c1c-ccab7bc94b1c [ 766.113026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Expecting reply to msg 88ca6a584ab9450588e677fb8b7d9864 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.134779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88ca6a584ab9450588e677fb8b7d9864 [ 766.217616] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397725, 'name': CreateVM_Task, 'duration_secs': 0.251131} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 766.217755] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 766.218274] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 766.218374] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 766.218631] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 766.218902] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5c1276eb-b222-4e9f-8c49-fa358d4b0a21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.223317] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 766.223317] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]523cf91e-cbbb-05d6-e27a-f41b3c30a8e7" [ 766.223317] env[62736]: _type = "Task" [ 766.223317] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 766.232959] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]523cf91e-cbbb-05d6-e27a-f41b3c30a8e7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 766.236636] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e437a92-2687-431a-b437-c6fa3b2d6b10 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.243318] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92c3f8a3-d3ee-4b0e-8d36-520eb89443cd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.275877] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-543c162c-825a-4752-be3f-1031ff3603d9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.281389] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-981a1060-6618-453d-a800-3494f103c1ce {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.286039] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Successfully created port: 7546ffee-78bf-462a-a94a-d087f5f1b40d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 766.296819] env[62736]: DEBUG nova.compute.provider_tree [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 766.297373] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg f75cdcb16cd644b5a4f9453b88d06f86 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.304350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f75cdcb16cd644b5a4f9453b88d06f86 [ 766.384539] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 766.386225] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg f7a681b7f6b541d1ba9f12664d8ba80d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.431065] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7a681b7f6b541d1ba9f12664d8ba80d [ 766.617401] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c9ea2d0a-832c-4d46-a223-77ca6d1193cf tempest-ServersV294TestFqdnHostnames-1389170272 tempest-ServersV294TestFqdnHostnames-1389170272-project-member] Lock "427400bc-f265-40cb-8c1c-ccab7bc94b1c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 143.092s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 766.618124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg db0db8ac05a54af18ce1c59ef0f1d3b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.626867] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db0db8ac05a54af18ce1c59ef0f1d3b4 [ 766.736186] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]523cf91e-cbbb-05d6-e27a-f41b3c30a8e7, 'name': SearchDatastore_Task, 'duration_secs': 0.011246} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 766.736594] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.736907] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 766.737196] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 766.737400] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 766.737731] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 766.737966] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a8a323f5-34fa-4f90-9c93-ceeb9e44af78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.745889] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 766.746128] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 766.746897] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-573d57d7-50a6-4578-92cf-7126fb2a04cd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.751862] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 766.751862] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52aeca24-33fb-bdc6-f59c-ea7643025fac" [ 766.751862] env[62736]: _type = "Task" [ 766.751862] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 766.761115] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52aeca24-33fb-bdc6-f59c-ea7643025fac, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 766.800858] env[62736]: DEBUG nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 766.805107] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2ed2db79687947789acdc3f46a7f6273 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.819302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ed2db79687947789acdc3f46a7f6273 [ 766.891010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg a07c3ebd4a2245a99d7f25f51831ec5b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 766.921244] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a07c3ebd4a2245a99d7f25f51831ec5b [ 767.061094] env[62736]: DEBUG nova.compute.manager [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Received event network-changed-7546ffee-78bf-462a-a94a-d087f5f1b40d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 767.061297] env[62736]: DEBUG nova.compute.manager [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Refreshing instance network info cache due to event network-changed-7546ffee-78bf-462a-a94a-d087f5f1b40d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 767.061514] env[62736]: DEBUG oslo_concurrency.lockutils [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] Acquiring lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 767.061681] env[62736]: DEBUG oslo_concurrency.lockutils [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] Acquired lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 767.061920] env[62736]: DEBUG nova.network.neutron [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Refreshing network info cache for port 7546ffee-78bf-462a-a94a-d087f5f1b40d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 767.062205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] Expecting reply to msg 081970da5c914534b9a35bdf40018d53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.069995] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 081970da5c914534b9a35bdf40018d53 [ 767.120465] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 767.122577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 160d0128d1fd497081ecb27c0439b430 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.198707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 160d0128d1fd497081ecb27c0439b430 [ 767.263390] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52aeca24-33fb-bdc6-f59c-ea7643025fac, 'name': SearchDatastore_Task, 'duration_secs': 0.008229} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 767.265550] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-03aaecc2-7eb6-47aa-8cb6-addfa77db56f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.271686] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 767.271686] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]525b0395-e3c6-dfd8-91b2-efb7ee31b6c0" [ 767.271686] env[62736]: _type = "Task" [ 767.271686] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 767.285518] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]525b0395-e3c6-dfd8-91b2-efb7ee31b6c0, 'name': SearchDatastore_Task, 'duration_secs': 0.008642} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 767.285790] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 767.286045] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 767.286285] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-bb113261-fb4c-456c-b2e4-e586d60b5e92 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.293273] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 767.293273] env[62736]: value = "task-397726" [ 767.293273] env[62736]: _type = "Task" [ 767.293273] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 767.301875] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397726, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 767.307755] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.928s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.308394] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Traceback (most recent call last): [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.driver.spawn(context, instance, image_meta, [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] vm_ref = self.build_virtual_machine(instance, [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] vif_infos = vmwarevif.get_vif_info(self._session, [ 767.308394] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] for vif in network_info: [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self._sync_wrapper(fn, *args, **kwargs) [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.wait() [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self[:] = self._gt.wait() [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self._exit_event.wait() [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] result = hub.switch() [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 767.308815] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return self.greenlet.switch() [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] result = function(*args, **kwargs) [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] return func(*args, **kwargs) [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise e [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] nwinfo = self.network_api.allocate_for_instance( [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] created_port_ids = self._update_ports_for_instance( [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] with excutils.save_and_reraise_exception(): [ 767.309254] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] self.force_reraise() [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise self.value [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] updated_port = self._update_port( [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] _ensure_no_port_binding_failure(port) [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] raise exception.PortBindingFailed(port_id=port['id']) [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] nova.exception.PortBindingFailed: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. [ 767.309732] env[62736]: ERROR nova.compute.manager [instance: d633de23-23ab-4297-b2d6-2702906b31a9] [ 767.310095] env[62736]: DEBUG nova.compute.utils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 767.310303] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.660s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.311838] env[62736]: INFO nova.compute.claims [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 767.313375] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 689253f287b847eb942f3f3b51f9a578 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.314630] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Build of instance d633de23-23ab-4297-b2d6-2702906b31a9 was re-scheduled: Binding failed for port fda4aa7d-b176-4a8a-8311-135c11345fee, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 767.315815] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 767.316047] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 767.316193] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 767.316347] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 767.316710] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 9bf570e6e06e413f8d761e6f91fe3984 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.324267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bf570e6e06e413f8d761e6f91fe3984 [ 767.325346] env[62736]: ERROR nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 767.325346] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.325346] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.325346] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.325346] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.325346] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.325346] env[62736]: ERROR nova.compute.manager raise self.value [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.325346] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 767.325346] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.325346] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 767.325877] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.325877] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 767.325877] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 767.325877] env[62736]: ERROR nova.compute.manager [ 767.325877] env[62736]: Traceback (most recent call last): [ 767.325877] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 767.325877] env[62736]: listener.cb(fileno) [ 767.325877] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 767.325877] env[62736]: result = function(*args, **kwargs) [ 767.325877] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 767.325877] env[62736]: return func(*args, **kwargs) [ 767.325877] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 767.325877] env[62736]: raise e [ 767.325877] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.325877] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 767.325877] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.325877] env[62736]: created_port_ids = self._update_ports_for_instance( [ 767.325877] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.325877] env[62736]: with excutils.save_and_reraise_exception(): [ 767.325877] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.325877] env[62736]: self.force_reraise() [ 767.325877] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.325877] env[62736]: raise self.value [ 767.325877] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.325877] env[62736]: updated_port = self._update_port( [ 767.325877] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.325877] env[62736]: _ensure_no_port_binding_failure(port) [ 767.325877] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.325877] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 767.326840] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 767.326840] env[62736]: Removing descriptor: 16 [ 767.363806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 689253f287b847eb942f3f3b51f9a578 [ 767.395587] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 767.423347] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 767.423662] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 767.423895] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 767.424149] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 767.424327] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 767.424562] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 767.424787] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 767.424946] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 767.425107] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 767.425262] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 767.425430] env[62736]: DEBUG nova.virt.hardware [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 767.426587] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4b43a47-0acc-425c-8bb8-75f340af153b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.436521] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-778abf93-b138-4ba0-9c87-ad1556405606 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.456632] env[62736]: ERROR nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Traceback (most recent call last): [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] yield resources [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.driver.spawn(context, instance, image_meta, [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] vm_ref = self.build_virtual_machine(instance, [ 767.456632] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] vif_infos = vmwarevif.get_vif_info(self._session, [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] for vif in network_info: [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return self._sync_wrapper(fn, *args, **kwargs) [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.wait() [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self[:] = self._gt.wait() [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return self._exit_event.wait() [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 767.457155] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] current.throw(*self._exc) [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] result = function(*args, **kwargs) [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return func(*args, **kwargs) [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise e [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] nwinfo = self.network_api.allocate_for_instance( [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] created_port_ids = self._update_ports_for_instance( [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] with excutils.save_and_reraise_exception(): [ 767.457590] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.force_reraise() [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise self.value [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] updated_port = self._update_port( [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] _ensure_no_port_binding_failure(port) [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise exception.PortBindingFailed(port_id=port['id']) [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 767.458000] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] [ 767.458000] env[62736]: INFO nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Terminating instance [ 767.459314] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquiring lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 767.584560] env[62736]: DEBUG nova.network.neutron [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 767.654495] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.728330] env[62736]: DEBUG nova.network.neutron [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.729991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] Expecting reply to msg baf7a985a7694ee0bd1ef04ef4947d52 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.739595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg baf7a985a7694ee0bd1ef04ef4947d52 [ 767.805280] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397726, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.425972} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 767.805544] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 767.805743] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 767.805983] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-3de2304a-60b3-49b0-9a24-7b00ef809481 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.812252] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 767.812252] env[62736]: value = "task-397727" [ 767.812252] env[62736]: _type = "Task" [ 767.812252] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 767.823607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg f161f702f2834787a8f7896610986e79 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 767.827265] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397727, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 767.833240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f161f702f2834787a8f7896610986e79 [ 767.853604] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 767.996367] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.996839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 8629ddc9310740ee903459be0a9aabb1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 768.004716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8629ddc9310740ee903459be0a9aabb1 [ 768.236318] env[62736]: DEBUG oslo_concurrency.lockutils [req-4412a6a1-867f-4d8c-a05c-5a24cad3a658 req-102a0cfa-0da9-4377-a3f4-4eeba82d5a00 service nova] Releasing lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 768.236746] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquired lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 768.236931] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 768.237551] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 1b7dc359d99a48f7984742ffd142d056 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 768.244350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b7dc359d99a48f7984742ffd142d056 [ 768.323061] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397727, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.064381} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 768.323324] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 768.324125] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69f01d1a-55fe-464a-a43c-b6d5f116bdb2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.351328] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfiguring VM instance instance-00000032 to attach disk [datastore2] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 768.353998] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-1dab45fa-42b9-4383-9b87-d0ec1b1a21c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.375322] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 768.375322] env[62736]: value = "task-397728" [ 768.375322] env[62736]: _type = "Task" [ 768.375322] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 768.388563] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397728, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 768.500188] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-d633de23-23ab-4297-b2d6-2702906b31a9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 768.500409] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 768.500582] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 768.500737] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 768.519256] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.520042] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2345b32d12f14f56b6cd7740dfd6e7b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 768.537354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2345b32d12f14f56b6cd7740dfd6e7b3 [ 768.654832] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05a39017-379d-40cb-bf49-a352725946be {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.662281] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-653f5a4f-5dd0-489a-abfa-d40904ccde2c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.693271] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb7a4c8f-ccc6-423f-a049-b3d55a0384f1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.700112] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4694c343-7a0c-4c9b-987f-1bc409d284bb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.715320] env[62736]: DEBUG nova.compute.provider_tree [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 768.715942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 669a689be3f5412ca16471f6d5c1c51a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 768.722888] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 669a689be3f5412ca16471f6d5c1c51a [ 768.753835] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.834809] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.835348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 0e27aba0c4dc447b9ffd539a65fd4b98 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 768.846035] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e27aba0c4dc447b9ffd539a65fd4b98 [ 768.886520] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397728, 'name': ReconfigVM_Task, 'duration_secs': 0.271396} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 768.886802] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfigured VM instance instance-00000032 to attach disk [datastore2] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 768.887470] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-a908b6f4-47e9-4ffe-a420-b81783f714e1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.894100] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 768.894100] env[62736]: value = "task-397729" [ 768.894100] env[62736]: _type = "Task" [ 768.894100] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 768.902127] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397729, 'name': Rename_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 769.021955] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.022532] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg a1e91e72f51c4fde85f6ad30ff31dd02 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.035976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1e91e72f51c4fde85f6ad30ff31dd02 [ 769.123039] env[62736]: DEBUG nova.compute.manager [req-a42a2db7-a073-44b7-9a7b-3c394d55a84a req-ea0d109b-66ae-455c-9e7f-c760a92deb2b service nova] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Received event network-vif-deleted-7546ffee-78bf-462a-a94a-d087f5f1b40d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 769.219154] env[62736]: DEBUG nova.scheduler.client.report [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 769.221930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 044bc6cca1564f42bfa4abc48998c326 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.257629] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 044bc6cca1564f42bfa4abc48998c326 [ 769.337602] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Releasing lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 769.338011] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 769.338204] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 769.338556] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-838e751b-2015-4583-bdf5-3acd7a568b3f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.348227] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa245a23-f56b-462c-9d32-b52dcd8bf18a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.371838] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3135faf8-b918-42f5-bcaa-c23a19166bb3 could not be found. [ 769.372075] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 769.372257] env[62736]: INFO nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Took 0.03 seconds to destroy the instance on the hypervisor. [ 769.372500] env[62736]: DEBUG oslo.service.loopingcall [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 769.372713] env[62736]: DEBUG nova.compute.manager [-] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 769.372807] env[62736]: DEBUG nova.network.neutron [-] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 769.388391] env[62736]: DEBUG nova.network.neutron [-] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 769.388881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 887e1571539f4e6385e11d1ec8e858db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.398422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 887e1571539f4e6385e11d1ec8e858db [ 769.404276] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397729, 'name': Rename_Task, 'duration_secs': 0.136774} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 769.404522] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 769.404747] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-40b02949-46d5-41cf-b42d-28fc17711f00 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.410701] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 769.410701] env[62736]: value = "task-397730" [ 769.410701] env[62736]: _type = "Task" [ 769.410701] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 769.418615] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397730, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 769.525394] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: d633de23-23ab-4297-b2d6-2702906b31a9] Took 1.02 seconds to deallocate network for instance. [ 769.527128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg e179da0b851548a59aa78b2b4c84b329 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.562198] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e179da0b851548a59aa78b2b4c84b329 [ 769.725785] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.415s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.726446] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 769.728164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 5037eb74fafa456d85860d1a6a00c371 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.729287] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.168s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.730537] env[62736]: INFO nova.compute.claims [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 769.731968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 74b4731e099841efab3ab84b7a55c5a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.760669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5037eb74fafa456d85860d1a6a00c371 [ 769.765562] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74b4731e099841efab3ab84b7a55c5a8 [ 769.890675] env[62736]: DEBUG nova.network.neutron [-] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 769.891310] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 53e6f3f470db44059d20ede9160a5b78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 769.900436] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53e6f3f470db44059d20ede9160a5b78 [ 769.920375] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397730, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 770.031170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 4445324fdca54e168e8b83ab3e5e80ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.062925] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4445324fdca54e168e8b83ab3e5e80ef [ 770.235186] env[62736]: DEBUG nova.compute.utils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 770.235851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg dec454817a984057beb49c5547294919 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.237968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg d64ebb5bdb4a4c2494779b9b713e2121 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.238949] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 770.239084] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 770.245975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d64ebb5bdb4a4c2494779b9b713e2121 [ 770.246522] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dec454817a984057beb49c5547294919 [ 770.295367] env[62736]: DEBUG nova.policy [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fe5ca027bccd496bbfb9957653cb612d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dc923c7bf786472c9d72f276a49e29fb', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 770.393753] env[62736]: INFO nova.compute.manager [-] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Took 1.02 seconds to deallocate network for instance. [ 770.396125] env[62736]: DEBUG nova.compute.claims [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 770.396301] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 770.420549] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397730, 'name': PowerOnVM_Task} progress is 100%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 770.555370] env[62736]: INFO nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Deleted allocations for instance d633de23-23ab-4297-b2d6-2702906b31a9 [ 770.561917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d4cd77ac30414f1eb4de0f339c0318c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.577284] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4cd77ac30414f1eb4de0f339c0318c8 [ 770.640582] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Successfully created port: 95eb6170-9245-415c-9bdb-5a5437c7ddb8 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 770.741692] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 770.741692] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 9ffe32f6c11d4e7bbdd4c5da026250f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.777812] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ffe32f6c11d4e7bbdd4c5da026250f2 [ 770.924163] env[62736]: DEBUG oslo_vmware.api [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397730, 'name': PowerOnVM_Task, 'duration_secs': 1.011867} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 770.927685] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 770.928061] env[62736]: DEBUG nova.compute.manager [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 770.929301] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aebab8d-1041-48c6-a15a-0aed5288d839 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.937378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 5a2d4ed985214396aae108fdfaab301c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 770.980114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a2d4ed985214396aae108fdfaab301c [ 771.059243] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67d81c4c-dc8b-4c1d-922e-5fa7e1eb50ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.063597] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "d633de23-23ab-4297-b2d6-2702906b31a9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 142.134s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 771.064189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 2e2f40194b124a619e3a34cf2bb00446 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.068133] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c70b184-5037-41a8-9a5f-6401e44a9b69 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.116098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e2f40194b124a619e3a34cf2bb00446 [ 771.116098] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e689b6c5-f57b-4291-ae60-10e5cbe7e1fd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.124321] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4362326a-fd51-4c97-9df1-bd09cd2a2b40 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.138610] env[62736]: DEBUG nova.compute.provider_tree [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 771.139113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 6d87e49a17714a269d03536d4d5664b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.146177] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d87e49a17714a269d03536d4d5664b2 [ 771.248813] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 8dce3768e33a4ee299acf7e53e0fbd6e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.283140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dce3768e33a4ee299acf7e53e0fbd6e [ 771.368122] env[62736]: DEBUG nova.compute.manager [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Received event network-changed-95eb6170-9245-415c-9bdb-5a5437c7ddb8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 771.368122] env[62736]: DEBUG nova.compute.manager [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Refreshing instance network info cache due to event network-changed-95eb6170-9245-415c-9bdb-5a5437c7ddb8. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 771.368122] env[62736]: DEBUG oslo_concurrency.lockutils [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] Acquiring lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 771.368122] env[62736]: DEBUG oslo_concurrency.lockutils [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] Acquired lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 771.368122] env[62736]: DEBUG nova.network.neutron [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Refreshing network info cache for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 771.368457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] Expecting reply to msg 3108cc4d9d3c43d294cbac3c98645ddf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.370298] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3108cc4d9d3c43d294cbac3c98645ddf [ 771.449266] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.551780] env[62736]: ERROR nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 771.551780] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 771.551780] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 771.551780] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 771.551780] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 771.551780] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 771.551780] env[62736]: ERROR nova.compute.manager raise self.value [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 771.551780] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 771.551780] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 771.551780] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 771.552195] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 771.552195] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 771.552195] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 771.552195] env[62736]: ERROR nova.compute.manager [ 771.552195] env[62736]: Traceback (most recent call last): [ 771.552195] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 771.552195] env[62736]: listener.cb(fileno) [ 771.552195] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 771.552195] env[62736]: result = function(*args, **kwargs) [ 771.552195] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 771.552195] env[62736]: return func(*args, **kwargs) [ 771.552195] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 771.552195] env[62736]: raise e [ 771.552195] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 771.552195] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 771.552195] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 771.552195] env[62736]: created_port_ids = self._update_ports_for_instance( [ 771.552195] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 771.552195] env[62736]: with excutils.save_and_reraise_exception(): [ 771.552195] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 771.552195] env[62736]: self.force_reraise() [ 771.552195] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 771.552195] env[62736]: raise self.value [ 771.552195] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 771.552195] env[62736]: updated_port = self._update_port( [ 771.552195] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 771.552195] env[62736]: _ensure_no_port_binding_failure(port) [ 771.552195] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 771.552195] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 771.552817] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 771.552817] env[62736]: Removing descriptor: 16 [ 771.571302] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 771.571302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg bf67440a53f0499eab89f02a297bcb43 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.604414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf67440a53f0499eab89f02a297bcb43 [ 771.643430] env[62736]: DEBUG nova.scheduler.client.report [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 771.652413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg ab3520e1786247319a1b81eb01e82b01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.663508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab3520e1786247319a1b81eb01e82b01 [ 771.756452] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg cfd8085a90084c8281480aaa892d6b06 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.757994] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 771.765802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfd8085a90084c8281480aaa892d6b06 [ 771.780017] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 771.780273] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 771.780428] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 771.780613] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 771.780738] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 771.781017] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 771.781260] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 771.781493] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 771.781652] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 771.781851] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 771.782058] env[62736]: DEBUG nova.virt.hardware [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 771.783150] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-964494fe-007c-4646-bf79-67a2fb9cb8e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.791925] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adfff6c3-2048-4e71-b8bf-c538c5c7c385 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 771.809766] env[62736]: ERROR nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Traceback (most recent call last): [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] yield resources [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.driver.spawn(context, instance, image_meta, [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] vm_ref = self.build_virtual_machine(instance, [ 771.809766] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] vif_infos = vmwarevif.get_vif_info(self._session, [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] for vif in network_info: [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return self._sync_wrapper(fn, *args, **kwargs) [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.wait() [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self[:] = self._gt.wait() [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return self._exit_event.wait() [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 771.810097] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] current.throw(*self._exc) [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] result = function(*args, **kwargs) [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return func(*args, **kwargs) [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise e [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] nwinfo = self.network_api.allocate_for_instance( [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] created_port_ids = self._update_ports_for_instance( [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] with excutils.save_and_reraise_exception(): [ 771.810402] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.force_reraise() [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise self.value [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] updated_port = self._update_port( [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] _ensure_no_port_binding_failure(port) [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise exception.PortBindingFailed(port_id=port['id']) [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 771.810696] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] [ 771.810696] env[62736]: INFO nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Terminating instance [ 771.812199] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 771.893405] env[62736]: DEBUG nova.network.neutron [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 771.961028] env[62736]: DEBUG nova.network.neutron [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.961577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] Expecting reply to msg f755bb960779487db5b109bc56b5cbdf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 771.970594] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f755bb960779487db5b109bc56b5cbdf [ 772.090993] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.154580] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.425s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.155112] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 772.157089] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg fdf9c4ebc7294d77ab6b9c5c58d56994 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 772.158479] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.617s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.160711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg b9bda0ae84b64d168663efb2bcc19add in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 772.196974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdf9c4ebc7294d77ab6b9c5c58d56994 [ 772.198438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9bda0ae84b64d168663efb2bcc19add [ 772.261183] env[62736]: INFO nova.compute.manager [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Rebuilding instance [ 772.298061] env[62736]: DEBUG nova.compute.manager [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 772.298937] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42996969-5826-4200-baa9-15c2818c5983 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.306304] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg 268eb83705d14eb38bb9c4edb9607d15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 772.387847] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 268eb83705d14eb38bb9c4edb9607d15 [ 772.464507] env[62736]: DEBUG oslo_concurrency.lockutils [req-d166cf80-d8de-44e7-b909-287652fe7016 req-7b9ac4f5-c636-4425-93e0-84a184abfc23 service nova] Releasing lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 772.464935] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquired lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 772.465116] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 772.465543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 3b9542a9f8094400a838242fe7e6aee1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 772.473473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b9542a9f8094400a838242fe7e6aee1 [ 772.660340] env[62736]: DEBUG nova.compute.utils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 772.660736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 5306cce56022493da17dbf6d48807e7b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 772.661660] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 772.661834] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 772.692567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5306cce56022493da17dbf6d48807e7b [ 772.736293] env[62736]: DEBUG nova.policy [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fe5ca027bccd496bbfb9957653cb612d', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dc923c7bf786472c9d72f276a49e29fb', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 772.808896] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 772.809288] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-8ff88f88-864b-4476-9f66-a80a1df3fe83 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.816377] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 772.816377] env[62736]: value = "task-397731" [ 772.816377] env[62736]: _type = "Task" [ 772.816377] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 772.826676] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397731, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 772.993557] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0f77a8d-b89e-42c6-8566-1ceb8b8ad192 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.997123] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 773.003406] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cacab5a-eaac-40d7-aa25-57111898aca6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.039234] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Successfully created port: 993db6d2-a560-4b87-9cd6-1bd027525b39 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 773.043245] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2cdca732-01e9-4ca4-ae3b-258df14a11c9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.050714] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f6958fb-dea7-4583-8063-0337a1170242 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.067117] env[62736]: DEBUG nova.compute.provider_tree [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 773.067900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 7e200990d8cd4e66ba78263d8fdcfce5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.075698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7e200990d8cd4e66ba78263d8fdcfce5 [ 773.091520] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.092214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg cf9fe87e80a342fea51e22958580606b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.103007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf9fe87e80a342fea51e22958580606b [ 773.168493] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 773.170623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 10f02646824a47a3a17e28f1620d1421 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.210876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10f02646824a47a3a17e28f1620d1421 [ 773.325200] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397731, 'name': PowerOffVM_Task, 'duration_secs': 0.205803} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 773.325573] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 773.325833] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 773.326588] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2afd1ce-66c7-48c9-9800-e7189617a710 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.332501] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 773.332743] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-90eff68c-116d-44bc-9299-6dc2989c1755 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.356739] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 773.357012] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleting contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 773.357270] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Deleting the datastore file [datastore2] bbc25649-cfa4-4f5f-ab1e-5f1347780612 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 773.357562] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-e7463a7d-0a2c-4f9f-909f-a644be0311b4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.363705] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 773.363705] env[62736]: value = "task-397733" [ 773.363705] env[62736]: _type = "Task" [ 773.363705] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 773.371403] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397733, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 773.392680] env[62736]: DEBUG nova.compute.manager [req-75cda059-811b-4d56-aa3a-b5b9a81727c2 req-06c1d7ea-5f64-4207-a72c-f2ae1636023f service nova] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Received event network-vif-deleted-95eb6170-9245-415c-9bdb-5a5437c7ddb8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 773.570419] env[62736]: DEBUG nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 773.572969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 4e84b3615bfc4762b45ec71b98c5b5b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.583844] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e84b3615bfc4762b45ec71b98c5b5b8 [ 773.593804] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Releasing lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 773.594195] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 773.594374] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 773.594638] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3146e18e-1d4e-4f66-9e2a-8088286e8697 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.603519] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7e93947-292f-461c-872c-eb63a276fc68 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 773.626613] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 97927074-a676-4ebd-aa7a-fe92e3ce1be6 could not be found. [ 773.626761] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 773.626964] env[62736]: INFO nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 773.627206] env[62736]: DEBUG oslo.service.loopingcall [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 773.627393] env[62736]: DEBUG nova.compute.manager [-] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 773.627481] env[62736]: DEBUG nova.network.neutron [-] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 773.647534] env[62736]: DEBUG nova.network.neutron [-] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 773.648050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 40afb03111de407bac62af1cc0fee810 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.654644] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40afb03111de407bac62af1cc0fee810 [ 773.678752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 69468f5730bd48e3bfbf79b3f9759577 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.710941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69468f5730bd48e3bfbf79b3f9759577 [ 773.863449] env[62736]: ERROR nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 773.863449] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 773.863449] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 773.863449] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 773.863449] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 773.863449] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 773.863449] env[62736]: ERROR nova.compute.manager raise self.value [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 773.863449] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 773.863449] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 773.863449] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 773.863867] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 773.863867] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 773.863867] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 773.863867] env[62736]: ERROR nova.compute.manager [ 773.863867] env[62736]: Traceback (most recent call last): [ 773.863867] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 773.863867] env[62736]: listener.cb(fileno) [ 773.863867] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 773.863867] env[62736]: result = function(*args, **kwargs) [ 773.863867] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 773.863867] env[62736]: return func(*args, **kwargs) [ 773.863867] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 773.863867] env[62736]: raise e [ 773.863867] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 773.863867] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 773.863867] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 773.863867] env[62736]: created_port_ids = self._update_ports_for_instance( [ 773.863867] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 773.863867] env[62736]: with excutils.save_and_reraise_exception(): [ 773.863867] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 773.863867] env[62736]: self.force_reraise() [ 773.863867] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 773.863867] env[62736]: raise self.value [ 773.863867] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 773.863867] env[62736]: updated_port = self._update_port( [ 773.863867] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 773.863867] env[62736]: _ensure_no_port_binding_failure(port) [ 773.863867] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 773.863867] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 773.864636] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 773.864636] env[62736]: Removing descriptor: 16 [ 773.873802] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397733, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.187224} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 773.874049] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 773.874231] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleted contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 773.874400] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 773.876254] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg 0eaa9c9f893441ebb63c9e1becdbcbc5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 773.922498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0eaa9c9f893441ebb63c9e1becdbcbc5 [ 774.075785] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.917s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 774.076441] env[62736]: ERROR nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Traceback (most recent call last): [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.driver.spawn(context, instance, image_meta, [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] vm_ref = self.build_virtual_machine(instance, [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] vif_infos = vmwarevif.get_vif_info(self._session, [ 774.076441] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] for vif in network_info: [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self._sync_wrapper(fn, *args, **kwargs) [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.wait() [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self[:] = self._gt.wait() [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self._exit_event.wait() [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] result = hub.switch() [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 774.076812] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return self.greenlet.switch() [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] result = function(*args, **kwargs) [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] return func(*args, **kwargs) [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise e [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] nwinfo = self.network_api.allocate_for_instance( [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] created_port_ids = self._update_ports_for_instance( [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] with excutils.save_and_reraise_exception(): [ 774.077138] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] self.force_reraise() [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise self.value [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] updated_port = self._update_port( [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] _ensure_no_port_binding_failure(port) [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] raise exception.PortBindingFailed(port_id=port['id']) [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] nova.exception.PortBindingFailed: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. [ 774.077475] env[62736]: ERROR nova.compute.manager [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] [ 774.077745] env[62736]: DEBUG nova.compute.utils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 774.078702] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Build of instance 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0 was re-scheduled: Binding failed for port 9ad2958c-98f6-40ec-8c0b-17728bf04579, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 774.079179] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 774.079437] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 774.079592] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 774.079749] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 774.080171] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg dd6ccaa2b15f45e0995b291affe34d79 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.081248] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 14.947s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 774.082105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 774e70ece5664e50b2da09f0691d3112 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.086875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd6ccaa2b15f45e0995b291affe34d79 [ 774.103768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 774e70ece5664e50b2da09f0691d3112 [ 774.150089] env[62736]: DEBUG nova.network.neutron [-] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.150557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 335c52d4f89940868c01b1e98a069c3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.157999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 335c52d4f89940868c01b1e98a069c3a [ 774.183830] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 774.207848] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 774.208198] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 774.208420] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 774.208665] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 774.208867] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 774.209084] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 774.209356] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 774.209637] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 774.209889] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 774.210115] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 774.210376] env[62736]: DEBUG nova.virt.hardware [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 774.211310] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d99b9451-51f5-438f-9dc2-d3a96e0c5403 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.219313] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19cc84eb-2a16-43bb-bf46-8a2f23b19c34 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.233471] env[62736]: ERROR nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Traceback (most recent call last): [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] yield resources [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.driver.spawn(context, instance, image_meta, [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] vm_ref = self.build_virtual_machine(instance, [ 774.233471] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] vif_infos = vmwarevif.get_vif_info(self._session, [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] for vif in network_info: [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return self._sync_wrapper(fn, *args, **kwargs) [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.wait() [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self[:] = self._gt.wait() [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return self._exit_event.wait() [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 774.233910] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] current.throw(*self._exc) [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] result = function(*args, **kwargs) [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return func(*args, **kwargs) [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise e [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] nwinfo = self.network_api.allocate_for_instance( [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] created_port_ids = self._update_ports_for_instance( [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] with excutils.save_and_reraise_exception(): [ 774.234213] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.force_reraise() [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise self.value [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] updated_port = self._update_port( [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] _ensure_no_port_binding_failure(port) [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise exception.PortBindingFailed(port_id=port['id']) [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 774.234507] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] [ 774.234507] env[62736]: INFO nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Terminating instance [ 774.236595] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 774.236836] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquired lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 774.237102] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 774.237585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 9163fb221b4f4c0d8e3fa625a28eec64 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.244111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9163fb221b4f4c0d8e3fa625a28eec64 [ 774.380513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg 676e45dc83aa43629b6b33f38ce87738 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.408276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 676e45dc83aa43629b6b33f38ce87738 [ 774.586473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 937d757e837649389eca13f6dde1434b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.595514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 937d757e837649389eca13f6dde1434b [ 774.599132] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.652624] env[62736]: INFO nova.compute.manager [-] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Took 1.02 seconds to deallocate network for instance. [ 774.654826] env[62736]: DEBUG nova.compute.claims [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 774.654987] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 774.673430] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.673918] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 26e81e6438ff4deabedf5b12867b5bc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.682087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26e81e6438ff4deabedf5b12867b5bc6 [ 774.754446] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.802355] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 774.802897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 854f51f9a1af4d2c89636fea26c41f5f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 774.810919] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 854f51f9a1af4d2c89636fea26c41f5f [ 774.903580] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 774.903826] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 774.903977] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 774.904171] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 774.904314] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 774.904458] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 774.904662] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 774.904818] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 774.904980] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 774.905136] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 774.905303] env[62736]: DEBUG nova.virt.hardware [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 774.906141] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eddce32d-6d5c-4071-9fd6-5b500f9058ff {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.913769] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-546700f0-0e83-4cd2-b910-09b0f1855c20 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.928076] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 774.933573] env[62736]: DEBUG oslo.service.loopingcall [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 774.933779] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 774.934022] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-42c768f0-152f-413c-b6a6-f69492ec53be {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 774.949985] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 774.949985] env[62736]: value = "task-397734" [ 774.949985] env[62736]: _type = "Task" [ 774.949985] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 774.956749] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397734, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 775.111572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 2a7e5e7c53de4537ba395ad24b6065a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.121990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a7e5e7c53de4537ba395ad24b6065a3 [ 775.175962] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 775.176227] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 775.176411] env[62736]: DEBUG nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 775.176668] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 775.192146] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 775.192715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 85d6c21ac15e4de389c50bab6e7aa173 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.199518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85d6c21ac15e4de389c50bab6e7aa173 [ 775.305449] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Releasing lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 775.305915] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 775.306074] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 775.306367] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5a65f169-44d1-440f-a0c0-9b88be4f9ccf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.316024] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab49406a-dd25-44d4-88dd-5fec57945859 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.338504] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 61f31838-2fd1-4804-82b7-56b3a08c3ff5 could not be found. [ 775.338504] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 775.338504] env[62736]: INFO nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 775.338504] env[62736]: DEBUG oslo.service.loopingcall [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 775.338504] env[62736]: DEBUG nova.compute.manager [-] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 775.338504] env[62736]: DEBUG nova.network.neutron [-] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 775.352311] env[62736]: DEBUG nova.network.neutron [-] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 775.352810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e0cbe7b218d6431ea2daf552dd4ff2d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.359192] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0cbe7b218d6431ea2daf552dd4ff2d5 [ 775.415886] env[62736]: DEBUG nova.compute.manager [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Received event network-changed-993db6d2-a560-4b87-9cd6-1bd027525b39 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 775.416119] env[62736]: DEBUG nova.compute.manager [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Refreshing instance network info cache due to event network-changed-993db6d2-a560-4b87-9cd6-1bd027525b39. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 775.416263] env[62736]: DEBUG oslo_concurrency.lockutils [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] Acquiring lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 775.416402] env[62736]: DEBUG oslo_concurrency.lockutils [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] Acquired lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 775.416559] env[62736]: DEBUG nova.network.neutron [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Refreshing network info cache for port 993db6d2-a560-4b87-9cd6-1bd027525b39 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 775.416960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] Expecting reply to msg c3bc7be7f23340b686510c488badb887 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.423496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3bc7be7f23340b686510c488badb887 [ 775.459165] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397734, 'name': CreateVM_Task, 'duration_secs': 0.239414} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 775.459336] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 775.459801] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 775.459962] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 775.460283] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 775.460525] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-88642590-be69-4297-ae97-7e545a189f26 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.464686] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 775.464686] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52f22148-728b-9c08-7105-dff4eef7166b" [ 775.464686] env[62736]: _type = "Task" [ 775.464686] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 775.471621] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52f22148-728b-9c08-7105-dff4eef7166b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 775.614657] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.614836] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance bbc25649-cfa4-4f5f-ab1e-5f1347780612 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.614960] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.615078] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 3135faf8-b918-42f5-bcaa-c23a19166bb3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.615191] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 97927074-a676-4ebd-aa7a-fe92e3ce1be6 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.615306] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 61f31838-2fd1-4804-82b7-56b3a08c3ff5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.615999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 86fd4e34e1e34e3894a7628fc296dca6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.625848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86fd4e34e1e34e3894a7628fc296dca6 [ 775.694965] env[62736]: DEBUG nova.network.neutron [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.695457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 6ef138c3c9c046bb8d98fcf89e0e6480 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.703483] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ef138c3c9c046bb8d98fcf89e0e6480 [ 775.854531] env[62736]: DEBUG nova.network.neutron [-] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.855069] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 502d1045d8bd48c0855a75aaee3e7e6b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.863202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 502d1045d8bd48c0855a75aaee3e7e6b [ 775.936048] env[62736]: DEBUG nova.network.neutron [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 775.974145] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52f22148-728b-9c08-7105-dff4eef7166b, 'name': SearchDatastore_Task, 'duration_secs': 0.00878} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 775.974450] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 775.974675] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 775.974904] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 775.975044] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 775.975213] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 775.975472] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-72f94652-1592-4d13-9927-85ca45487536 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.981494] env[62736]: DEBUG nova.network.neutron [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 775.981972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] Expecting reply to msg 9e179cafb4314695941855b317e7b63b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 775.984301] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 775.984481] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 775.985158] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-22fa46a9-b833-4b87-966a-e8cf14699445 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.989767] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 775.989767] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52d979bc-6960-9931-cfb3-bac98874d31b" [ 775.989767] env[62736]: _type = "Task" [ 775.989767] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 775.993403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e179cafb4314695941855b317e7b63b [ 775.998849] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52d979bc-6960-9931-cfb3-bac98874d31b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 776.118356] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.118942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 64a5608da1eb476f851b51b1480cb0d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 776.129176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64a5608da1eb476f851b51b1480cb0d5 [ 776.197274] env[62736]: INFO nova.compute.manager [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0] Took 1.02 seconds to deallocate network for instance. [ 776.198961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg eaa2f08ff5c449d9b143a7e5d3703377 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 776.234852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eaa2f08ff5c449d9b143a7e5d3703377 [ 776.358522] env[62736]: INFO nova.compute.manager [-] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Took 1.02 seconds to deallocate network for instance. [ 776.360065] env[62736]: DEBUG nova.compute.claims [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 776.360245] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 776.485050] env[62736]: DEBUG oslo_concurrency.lockutils [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] Releasing lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 776.485050] env[62736]: DEBUG nova.compute.manager [req-502a746f-510f-4cc2-9226-7471eb87b91e req-16aec519-95f1-47ec-9d69-900ac53993ea service nova] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Received event network-vif-deleted-993db6d2-a560-4b87-9cd6-1bd027525b39 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 776.501289] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52d979bc-6960-9931-cfb3-bac98874d31b, 'name': SearchDatastore_Task, 'duration_secs': 0.009785} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 776.502141] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dab44c7d-1e0f-4317-99f8-e5745a108bf3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.507489] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 776.507489] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5286fde0-e410-6130-9026-2c03263cd290" [ 776.507489] env[62736]: _type = "Task" [ 776.507489] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 776.515838] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5286fde0-e410-6130-9026-2c03263cd290, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 776.621949] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance fd2547e6-9449-4839-8ebb-f02f01a85b8c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.622545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3eb54a4790844984bc24939591474c3c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 776.632608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3eb54a4790844984bc24939591474c3c [ 776.703171] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 4e5d91834cc446d0be1553c639670654 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 776.732055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e5d91834cc446d0be1553c639670654 [ 777.018306] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5286fde0-e410-6130-9026-2c03263cd290, 'name': SearchDatastore_Task, 'duration_secs': 0.009096} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 777.018700] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 777.019035] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 777.019340] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0609e956-d87e-4f1a-b28a-fc6966151480 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.025466] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 777.025466] env[62736]: value = "task-397735" [ 777.025466] env[62736]: _type = "Task" [ 777.025466] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 777.033400] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397735, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 777.125470] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 5014a870-32fd-4166-9446-c41ed2a4954e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 777.126085] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg fef3a64707624146978d21ef4a6a7efa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 777.136552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fef3a64707624146978d21ef4a6a7efa [ 777.224886] env[62736]: INFO nova.scheduler.client.report [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Deleted allocations for instance 88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0 [ 777.232030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2f9e7ccb40af42948368c12cb4efefe3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 777.246848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f9e7ccb40af42948368c12cb4efefe3 [ 777.536946] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397735, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.437867} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 777.537279] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 777.537501] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 777.537758] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-fac867ab-e759-444e-89cc-6ddd7a27f9c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 777.543446] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 777.543446] env[62736]: value = "task-397736" [ 777.543446] env[62736]: _type = "Task" [ 777.543446] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 777.551612] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397736, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 777.629664] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance b49cde53-0ef7-44a4-b894-e37d1cbf41a7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 777.630278] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg c4c78165800c4cec96cb1e43af433e9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 777.641636] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4c78165800c4cec96cb1e43af433e9a [ 777.734566] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8b33de38-de92-45c0-80f7-7424439a9c46 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "88c5a4ae-ab48-4873-8a42-ecc6d0f99dd0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 148.764s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.735184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 0bb1b49c3dd64cf78b932921fdcff3a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 777.744659] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bb1b49c3dd64cf78b932921fdcff3a8 [ 778.053489] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397736, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.056372} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 778.053874] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 778.054698] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-784aead8-663a-4e8c-8e86-335fd3b2f5bc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 778.073539] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfiguring VM instance instance-00000032 to attach disk [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 778.074033] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-d04dd773-7409-4965-b64b-f4b693b94033 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 778.092033] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 778.092033] env[62736]: value = "task-397737" [ 778.092033] env[62736]: _type = "Task" [ 778.092033] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 778.099326] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397737, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 778.134705] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 25e906f6-ec98-4c99-83c7-43f94c5790dd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 778.135378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 309974832dc840dda8a15798f0273268 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 778.147506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 309974832dc840dda8a15798f0273268 [ 778.237841] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 778.239685] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg e22c0cef4243447088a06dce6e589dc3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 778.278800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e22c0cef4243447088a06dce6e589dc3 [ 778.601936] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397737, 'name': ReconfigVM_Task, 'duration_secs': 0.48254} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 778.602294] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Reconfigured VM instance instance-00000032 to attach disk [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612/bbc25649-cfa4-4f5f-ab1e-5f1347780612.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 778.602968] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-c585fd09-6b72-4485-8bbc-32a9da64e958 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 778.608623] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 778.608623] env[62736]: value = "task-397738" [ 778.608623] env[62736]: _type = "Task" [ 778.608623] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 778.617736] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397738, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 778.638797] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 99735498-3c85-4a01-bc05-4def014618a8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 778.639592] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 15ce5bcc1afc4925993f4318dbe5b509 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 778.651358] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 15ce5bcc1afc4925993f4318dbe5b509 [ 778.759149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.118849] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397738, 'name': Rename_Task, 'duration_secs': 0.127589} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 779.119241] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 779.119718] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-420fd75a-035a-43a7-bec9-8fc31296bd9c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.126879] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Waiting for the task: (returnval){ [ 779.126879] env[62736]: value = "task-397739" [ 779.126879] env[62736]: _type = "Task" [ 779.126879] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 779.135936] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397739, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 779.141896] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 6c4ac72f-1636-4f4c-928c-0a6fe895ce37 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 779.142676] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg f17087f2c73a4d1ab0b576d07085dc2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 779.158567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f17087f2c73a4d1ab0b576d07085dc2b [ 779.638401] env[62736]: DEBUG oslo_vmware.api [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Task: {'id': task-397739, 'name': PowerOnVM_Task, 'duration_secs': 0.433615} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 779.638401] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 779.638401] env[62736]: DEBUG nova.compute.manager [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 779.643276] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e970ed6-5cc9-44e2-b43f-0271ddc4ab49 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 779.645094] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 451b9e70-f202-4a10-8b19-34b6167ef14d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 779.645737] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 59f67e0561954b3283dd3069043bd4cd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 779.649670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg f6eca4ac22d149d8b811f427aa624828 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 779.661792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59f67e0561954b3283dd3069043bd4cd [ 779.686567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6eca4ac22d149d8b811f427aa624828 [ 779.849097] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.849148] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 779.872136] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "1f90a84e-6c32-4846-8908-128323e33e98" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.872381] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "1f90a84e-6c32-4846-8908-128323e33e98" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.149738] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 780.150308] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 69eb982fe98b4e9e9727d6e4d762258a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 780.159831] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.163750] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69eb982fe98b4e9e9727d6e4d762258a [ 780.656388] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance ee85f964-db45-404c-9016-b618748ec63c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 780.656748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 9b2fbe95ef214735aa413047dfedc6cc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 780.669971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b2fbe95ef214735aa413047dfedc6cc [ 780.727423] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg a55b7a8ebf004078a38e48ef15a37aa6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 780.737014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a55b7a8ebf004078a38e48ef15a37aa6 [ 781.160051] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance d7947374-0e8f-4d00-a430-923f0f2bb858 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 781.160747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 9ef53d11debc405890029345187a9305 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 781.170423] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ef53d11debc405890029345187a9305 [ 781.229454] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 781.229761] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 781.229971] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 781.230152] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 781.230336] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 781.232396] env[62736]: INFO nova.compute.manager [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Terminating instance [ 781.234285] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "refresh_cache-bbc25649-cfa4-4f5f-ab1e-5f1347780612" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 781.234285] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquired lock "refresh_cache-bbc25649-cfa4-4f5f-ab1e-5f1347780612" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 781.234402] env[62736]: DEBUG nova.network.neutron [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 781.234777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 9d1c57272a9444a2b83c05d76259ebce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 781.240989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9d1c57272a9444a2b83c05d76259ebce [ 781.663841] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 781.664444] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 765b043058a44fbcae52e3b99ff59897 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 781.674563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 765b043058a44fbcae52e3b99ff59897 [ 781.752864] env[62736]: DEBUG nova.network.neutron [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 781.808815] env[62736]: DEBUG nova.network.neutron [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 781.809348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 8f9fde675b5543ee81be04ccaf356e14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 781.817114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f9fde675b5543ee81be04ccaf356e14 [ 782.167674] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance d866dcf0-c08b-4b12-be83-8de3e69c439f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 782.168404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 2d701074cb674f5390190a612e0d0f90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 782.178333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d701074cb674f5390190a612e0d0f90 [ 782.312034] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Releasing lock "refresh_cache-bbc25649-cfa4-4f5f-ab1e-5f1347780612" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 782.312445] env[62736]: DEBUG nova.compute.manager [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 782.312635] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 782.313492] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c8a4401-f371-4375-93b5-445afc6877b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.321195] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 782.321410] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-581f6cfb-7d21-445c-a476-7aaa151d970b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.327144] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 782.327144] env[62736]: value = "task-397740" [ 782.327144] env[62736]: _type = "Task" [ 782.327144] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 782.334371] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397740, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 782.671378] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance ddcf62db-af40-49d2-9465-66a10f0f2e2d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 782.671968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 8d0a9b3c565b4cf9acea0e93f2399bf7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 782.682228] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d0a9b3c565b4cf9acea0e93f2399bf7 [ 782.837348] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397740, 'name': PowerOffVM_Task, 'duration_secs': 0.185494} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 782.837617] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 782.837786] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 782.838020] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0b9709fa-087a-486d-8b9f-9b578034da02 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.862071] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 782.862281] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 782.862460] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Deleting the datastore file [datastore1] bbc25649-cfa4-4f5f-ab1e-5f1347780612 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 782.862701] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-46beabd8-379d-49e4-979a-9e7a21d26476 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 782.868507] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for the task: (returnval){ [ 782.868507] env[62736]: value = "task-397742" [ 782.868507] env[62736]: _type = "Task" [ 782.868507] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 782.875903] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397742, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 783.174866] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 783.175131] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 783.175279] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 783.378398] env[62736]: DEBUG oslo_vmware.api [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Task: {'id': task-397742, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.093503} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 783.380753] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 783.381030] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 783.381271] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 783.381537] env[62736]: INFO nova.compute.manager [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Took 1.07 seconds to destroy the instance on the hypervisor. [ 783.381858] env[62736]: DEBUG oslo.service.loopingcall [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 783.382256] env[62736]: DEBUG nova.compute.manager [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 783.382412] env[62736]: DEBUG nova.network.neutron [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 783.402226] env[62736]: DEBUG nova.network.neutron [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 783.402898] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ed0f855ccab34b55ace99c0c3e0dba82 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 783.411401] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed0f855ccab34b55ace99c0c3e0dba82 [ 783.422631] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4af7191a-0638-42b5-9b50-2e814682f62f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.428693] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0674f610-2258-443a-9149-6a459808232d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.458659] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c472cbc-ebcc-442e-8c89-fd987b201abd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.466431] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78eb5fa2-2c9f-4043-a751-81f32ce98902 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.478926] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 783.479417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg d71a9f0a24b5481db5c120db708f9130 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 783.486199] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d71a9f0a24b5481db5c120db708f9130 [ 783.905346] env[62736]: DEBUG nova.network.neutron [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 783.905909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1f2030f1aed8444996f2fe9a56cafb23 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 783.914407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f2030f1aed8444996f2fe9a56cafb23 [ 783.981536] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 783.984010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 22770a6d3a80425db36ea79de166ae9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 783.998034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22770a6d3a80425db36ea79de166ae9f [ 784.408130] env[62736]: INFO nova.compute.manager [-] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Took 1.03 seconds to deallocate network for instance. [ 784.412325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 2d06e8078b484158b4e958f6de6915a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 784.442422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d06e8078b484158b4e958f6de6915a9 [ 784.486133] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 784.486380] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 10.405s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 784.486653] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.389s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 784.488271] env[62736]: INFO nova.compute.claims [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 784.489924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 2ab4d979fd1144faa87f3d7396747c00 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 784.491105] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 784.491251] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Cleaning up deleted instances {{(pid=62736) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11226}} [ 784.491768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 16b2b2b525144d52ad1c2863a900a6b6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 784.513041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16b2b2b525144d52ad1c2863a900a6b6 [ 784.529532] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ab4d979fd1144faa87f3d7396747c00 [ 784.914819] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 784.995640] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] There are 3 instances to clean {{(pid=62736) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11235}} [ 784.995950] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 13247e87-08ab-434e-b091-8b6116280fb7] Instance has had 0 of 5 cleanup attempts {{(pid=62736) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11239}} [ 784.996988] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 94994ac179e74c829c6be53aab01c7df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 784.999052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 14571fc466b8481daf7e4cb200cd5ff9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 785.011014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14571fc466b8481daf7e4cb200cd5ff9 [ 785.038236] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94994ac179e74c829c6be53aab01c7df [ 785.500895] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: b42fe7a4-ec30-441f-b5dc-1f705ee5c881] Instance has had 0 of 5 cleanup attempts {{(pid=62736) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11239}} [ 785.501968] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 9db98ad873db4ba589b754889cba9740 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 785.526347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9db98ad873db4ba589b754889cba9740 [ 785.896507] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc165e7f-153c-429f-b9b0-fb19d5010b4d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.904346] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57f65d40-6ea4-4ed4-a861-e6ede4e99cc3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.933930] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e308a398-9c62-40cd-ad02-878f915cd167 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.940953] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04ba8d96-2dd1-4ac5-ba4f-82499aea2844 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.957004] env[62736]: DEBUG nova.compute.provider_tree [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 785.957568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 6af86c16ac6b4334991303d8a746edcc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 785.964659] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6af86c16ac6b4334991303d8a746edcc [ 786.004477] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 5228dcb4-ec4b-4cf3-b0cb-1f53adde9b91] Instance has had 0 of 5 cleanup attempts {{(pid=62736) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11239}} [ 786.005627] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg e02fae599d7645d5a65d3c96a1b42dd2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 786.027873] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e02fae599d7645d5a65d3c96a1b42dd2 [ 786.460490] env[62736]: DEBUG nova.scheduler.client.report [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 786.462908] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg fe1534c8f94f466d86a92fd0ec236759 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 786.474405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe1534c8f94f466d86a92fd0ec236759 [ 786.513038] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 786.513221] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Cleaning up deleted instances with incomplete migration {{(pid=62736) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11264}} [ 786.513543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 92630e4442b64827bec90005e854056b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 786.523366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92630e4442b64827bec90005e854056b [ 786.965965] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.479s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.966255] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 786.967827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg af4c7f49a5724633a092afb0f75086ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 786.968928] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 23.355s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 786.970701] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg e18c0539fad1411892fc01c50cbb98ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 786.998145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af4c7f49a5724633a092afb0f75086ff [ 787.000402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e18c0539fad1411892fc01c50cbb98ef [ 787.015606] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 787.015924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 65e09da7337142298764faed8d985109 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 787.022603] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65e09da7337142298764faed8d985109 [ 787.474664] env[62736]: DEBUG nova.compute.utils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 787.475309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg c28fd13ddc50456788f1ab48e884f10b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 787.479130] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 787.479290] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 787.484965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c28fd13ddc50456788f1ab48e884f10b [ 787.493061] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Refreshing inventories for resource provider 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:818}} [ 787.506914] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Updating ProviderTree inventory for provider 0c9afe22-9d34-458c-8118-58661faecbae from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:782}} [ 787.507131] env[62736]: DEBUG nova.compute.provider_tree [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Updating inventory in ProviderTree for provider 0c9afe22-9d34-458c-8118-58661faecbae with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 787.518280] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Refreshing aggregate associations for resource provider 0c9afe22-9d34-458c-8118-58661faecbae, aggregates: None {{(pid=62736) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:827}} [ 787.523692] env[62736]: DEBUG nova.policy [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c43c33165931465494c0a7da171126e4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de11ac365f704c2986c782a5918f802f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 787.538452] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Refreshing trait associations for resource provider 0c9afe22-9d34-458c-8118-58661faecbae, traits: COMPUTE_IMAGE_TYPE_ISO,COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_NODE {{(pid=62736) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:839}} [ 787.791584] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Successfully created port: 1a7e7d7a-929e-4293-87d4-20021d6f1e74 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 787.806225] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-270b43d7-7fcb-4cb7-a753-f80ae4047b91 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.814084] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0e7e7af-632e-4062-b33e-a9a7bf52fab6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.845154] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac973401-2fc0-4fd1-ab22-10742ed8d264 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.852871] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d15d8816-37c7-47ff-bb9a-bde29a002ee7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.866843] env[62736]: DEBUG nova.compute.provider_tree [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 787.867403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg bfb2a0d63eff4b2fbb9183002e867cf4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 787.875424] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfb2a0d63eff4b2fbb9183002e867cf4 [ 787.979972] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 787.981764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 838210b75135410b8b418c200b5c2a90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.015348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 838210b75135410b8b418c200b5c2a90 [ 788.374055] env[62736]: DEBUG nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 788.374055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 75ec42253d7a40d5b5695856238322bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.385387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75ec42253d7a40d5b5695856238322bd [ 788.465846] env[62736]: DEBUG nova.compute.manager [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Received event network-changed-1a7e7d7a-929e-4293-87d4-20021d6f1e74 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 788.466031] env[62736]: DEBUG nova.compute.manager [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Refreshing instance network info cache due to event network-changed-1a7e7d7a-929e-4293-87d4-20021d6f1e74. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 788.466241] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] Acquiring lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 788.466384] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] Acquired lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 788.466535] env[62736]: DEBUG nova.network.neutron [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Refreshing network info cache for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 788.467010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] Expecting reply to msg e015ad6741f44fb993e024cce3295183 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.473758] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e015ad6741f44fb993e024cce3295183 [ 788.485764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 9abcac0950bf4c84b2be5bd5835b942e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.515612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9abcac0950bf4c84b2be5bd5835b942e [ 788.597796] env[62736]: ERROR nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 788.597796] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 788.597796] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 788.597796] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 788.597796] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 788.597796] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 788.597796] env[62736]: ERROR nova.compute.manager raise self.value [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 788.597796] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 788.597796] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 788.597796] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 788.598210] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 788.598210] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 788.598210] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 788.598210] env[62736]: ERROR nova.compute.manager [ 788.598210] env[62736]: Traceback (most recent call last): [ 788.598210] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 788.598210] env[62736]: listener.cb(fileno) [ 788.598210] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 788.598210] env[62736]: result = function(*args, **kwargs) [ 788.598210] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 788.598210] env[62736]: return func(*args, **kwargs) [ 788.598210] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 788.598210] env[62736]: raise e [ 788.598210] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 788.598210] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 788.598210] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 788.598210] env[62736]: created_port_ids = self._update_ports_for_instance( [ 788.598210] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 788.598210] env[62736]: with excutils.save_and_reraise_exception(): [ 788.598210] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 788.598210] env[62736]: self.force_reraise() [ 788.598210] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 788.598210] env[62736]: raise self.value [ 788.598210] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 788.598210] env[62736]: updated_port = self._update_port( [ 788.598210] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 788.598210] env[62736]: _ensure_no_port_binding_failure(port) [ 788.598210] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 788.598210] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 788.599119] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 788.599119] env[62736]: Removing descriptor: 16 [ 788.876113] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.907s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 788.876861] env[62736]: ERROR nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Traceback (most recent call last): [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.driver.spawn(context, instance, image_meta, [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self._vmops.spawn(context, instance, image_meta, injected_files, [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] vm_ref = self.build_virtual_machine(instance, [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] vif_infos = vmwarevif.get_vif_info(self._session, [ 788.876861] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] for vif in network_info: [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self._sync_wrapper(fn, *args, **kwargs) [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.wait() [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self[:] = self._gt.wait() [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self._exit_event.wait() [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] result = hub.switch() [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 788.877130] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return self.greenlet.switch() [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] result = function(*args, **kwargs) [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] return func(*args, **kwargs) [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise e [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] nwinfo = self.network_api.allocate_for_instance( [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] created_port_ids = self._update_ports_for_instance( [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] with excutils.save_and_reraise_exception(): [ 788.877482] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] self.force_reraise() [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise self.value [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] updated_port = self._update_port( [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] _ensure_no_port_binding_failure(port) [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] raise exception.PortBindingFailed(port_id=port['id']) [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] nova.exception.PortBindingFailed: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. [ 788.877780] env[62736]: ERROR nova.compute.manager [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] [ 788.878059] env[62736]: DEBUG nova.compute.utils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 788.878847] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.116s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 788.880409] env[62736]: INFO nova.compute.claims [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 788.882160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 24f3e2851fd94c97beb52187e54756ad in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.883340] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Build of instance 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca was re-scheduled: Binding failed for port 21f35458-40bc-40fc-8381-9814ec25b9cb, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 788.883822] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 788.884067] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 788.884216] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquired lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 788.884368] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 788.884713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 572fb73b69754652a4627bb1e2dc90b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 788.891440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 572fb73b69754652a4627bb1e2dc90b2 [ 788.915964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24f3e2851fd94c97beb52187e54756ad [ 788.982929] env[62736]: DEBUG nova.network.neutron [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 788.988400] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 789.012566] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 789.012843] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 789.013004] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 789.013220] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 789.013395] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 789.013546] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 789.013750] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 789.013902] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 789.014062] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 789.014217] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 789.014388] env[62736]: DEBUG nova.virt.hardware [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 789.015238] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-455f2628-f40c-4cd3-86d4-d0ee68b9521e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.024025] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38ad43e0-b9b2-4906-a3b0-74da8f5e9c60 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.037921] env[62736]: ERROR nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Traceback (most recent call last): [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] yield resources [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.driver.spawn(context, instance, image_meta, [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] vm_ref = self.build_virtual_machine(instance, [ 789.037921] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] vif_infos = vmwarevif.get_vif_info(self._session, [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] for vif in network_info: [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return self._sync_wrapper(fn, *args, **kwargs) [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.wait() [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self[:] = self._gt.wait() [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return self._exit_event.wait() [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 789.038233] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] current.throw(*self._exc) [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] result = function(*args, **kwargs) [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return func(*args, **kwargs) [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise e [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] nwinfo = self.network_api.allocate_for_instance( [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] created_port_ids = self._update_ports_for_instance( [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] with excutils.save_and_reraise_exception(): [ 789.038533] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.force_reraise() [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise self.value [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] updated_port = self._update_port( [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] _ensure_no_port_binding_failure(port) [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise exception.PortBindingFailed(port_id=port['id']) [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 789.038829] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] [ 789.038829] env[62736]: INFO nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Terminating instance [ 789.041856] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 789.076455] env[62736]: DEBUG nova.network.neutron [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 789.076970] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] Expecting reply to msg d5281d526835438b800dfc935851b950 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 789.084801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5281d526835438b800dfc935851b950 [ 789.388698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 123569b17099490a984da98162790da4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 789.397004] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 123569b17099490a984da98162790da4 [ 789.404215] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 789.484276] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 789.484790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 3aa13311c01243afa19dc5b4dfb2dcd3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 789.492432] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3aa13311c01243afa19dc5b4dfb2dcd3 [ 789.579016] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7559461-5397-4dac-88a7-4c43d7aca03a req-f3693957-625c-4d11-8998-340d0f2e5dbe service nova] Releasing lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 789.579448] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 789.579638] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 789.580072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 525abe7382e54e988d174223e254e65a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 789.586489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 525abe7382e54e988d174223e254e65a [ 789.986464] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Releasing lock "refresh_cache-6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 789.986730] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 789.986893] env[62736]: DEBUG nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 789.987062] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 790.006153] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 790.007241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 02ef1e232cf34649a8fdc7a99661a149 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.016439] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02ef1e232cf34649a8fdc7a99661a149 [ 790.097507] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 790.135616] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e21fc5a-27e6-4901-9ba4-6b15540bddbf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.143243] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bd74616-5295-443d-952b-6b147355e853 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.176426] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 790.177050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 6f3d645cf416410aaa9e891e93f95570 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.181545] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-366b8e3f-3bda-4dec-ae00-53c7b569e120 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.185810] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b08aafd-8715-47f8-9cdf-a84237c7a4f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.190161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f3d645cf416410aaa9e891e93f95570 [ 790.201517] env[62736]: DEBUG nova.compute.provider_tree [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 790.202213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 18a2cb887f7b4fdbbdaf36ec11db0b07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.209200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18a2cb887f7b4fdbbdaf36ec11db0b07 [ 790.492954] env[62736]: DEBUG nova.compute.manager [req-9b71a4a6-76b6-4dba-ab62-1de8e90269f4 req-35c50130-5476-4c92-8724-bcd273854a88 service nova] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Received event network-vif-deleted-1a7e7d7a-929e-4293-87d4-20021d6f1e74 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 790.509207] env[62736]: DEBUG nova.network.neutron [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 790.509207] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg c28bd6a500db4a70bd8d8da85b2e1d4f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.516642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c28bd6a500db4a70bd8d8da85b2e1d4f [ 790.682724] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 790.682724] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 790.682862] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 790.683140] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-98e35462-5203-4f24-ba54-8c5679229318 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.693547] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e918ef2-f3ba-4619-af3b-ba961f488d38 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.704428] env[62736]: DEBUG nova.scheduler.client.report [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 790.706704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 95ec490b6d4f47179fa227c7e9326210 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.718883] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0 could not be found. [ 790.719093] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 790.719271] env[62736]: INFO nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 790.719528] env[62736]: DEBUG oslo.service.loopingcall [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 790.720255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95ec490b6d4f47179fa227c7e9326210 [ 790.720519] env[62736]: DEBUG nova.compute.manager [-] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 790.720617] env[62736]: DEBUG nova.network.neutron [-] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 790.736581] env[62736]: DEBUG nova.network.neutron [-] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 790.737181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f6bf89265d144edaa8dc125a2108b747 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 790.743708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6bf89265d144edaa8dc125a2108b747 [ 791.013921] env[62736]: INFO nova.compute.manager [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca] Took 1.02 seconds to deallocate network for instance. [ 791.013921] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg b184890ad16e479ba0679f842388ba61 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.048544] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b184890ad16e479ba0679f842388ba61 [ 791.210792] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.331s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 791.210792] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 791.212086] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 30d5dbdfcda141f6afef370fabbca656 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.213145] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.456s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 791.214601] env[62736]: INFO nova.compute.claims [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 791.216227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 41793842fda84d63b75ee2079ff79258 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.240484] env[62736]: DEBUG nova.network.neutron [-] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 791.240484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 80e24865628a4b8189b22eec94a7ee39 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.248026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30d5dbdfcda141f6afef370fabbca656 [ 791.249323] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41793842fda84d63b75ee2079ff79258 [ 791.250714] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80e24865628a4b8189b22eec94a7ee39 [ 791.517014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 3876234f30224da9a5f0c1b2d7ecf609 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.550030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3876234f30224da9a5f0c1b2d7ecf609 [ 791.719149] env[62736]: DEBUG nova.compute.utils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 791.719799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 83367236281d464b9c14aa3fd6ac565b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.720772] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 791.720934] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 791.723928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 7993419b55f54142b62bcb1d4a5d0449 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 791.733205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7993419b55f54142b62bcb1d4a5d0449 [ 791.741912] env[62736]: INFO nova.compute.manager [-] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Took 1.02 seconds to deallocate network for instance. [ 791.743926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83367236281d464b9c14aa3fd6ac565b [ 791.743989] env[62736]: DEBUG nova.compute.claims [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 791.744144] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 791.770981] env[62736]: DEBUG nova.policy [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd65b3ab163145e1bf370aa0dc63f65c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8003d16b1cc048bc94407d028838a4f6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 792.040273] env[62736]: INFO nova.scheduler.client.report [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Deleted allocations for instance 6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca [ 792.046973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 2f828226e7d34c3ca76734944eead454 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.074841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f828226e7d34c3ca76734944eead454 [ 792.106048] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Successfully created port: e0c0a849-c8ce-4d35-a085-a6276922400b {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 792.226128] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 792.228501] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg a4d63c4eea2e4d208677b369e15db611 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.267549] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4d63c4eea2e4d208677b369e15db611 [ 792.549649] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39f2fd73-1767-4df0-93d1-b4949fa249a4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.552842] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7744b891-7149-4f81-a9d6-800f73ed441e tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "6a7e7ba2-a972-48bd-83b7-8d1b6a3289ca" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 149.597s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.553366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 97d5ed3429c749c9aa5b7d9597017c9e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.560445] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f82864b-800b-4a7c-b868-2ae0527c2860 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.590475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97d5ed3429c749c9aa5b7d9597017c9e [ 792.591395] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ac81610-64db-47c6-b320-4fb0261cc226 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.599431] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aaf15014-9699-4b2c-9db4-ca645a5023e6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 792.613227] env[62736]: DEBUG nova.compute.provider_tree [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 792.613738] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 018fb4e3085c41bcbe299fa751f9c2d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.623611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 018fb4e3085c41bcbe299fa751f9c2d9 [ 792.737587] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg faaa87dec3674ef78986bad2f8037225 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.775032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg faaa87dec3674ef78986bad2f8037225 [ 792.969738] env[62736]: DEBUG nova.compute.manager [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Received event network-changed-e0c0a849-c8ce-4d35-a085-a6276922400b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 792.969919] env[62736]: DEBUG nova.compute.manager [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Refreshing instance network info cache due to event network-changed-e0c0a849-c8ce-4d35-a085-a6276922400b. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 792.970124] env[62736]: DEBUG oslo_concurrency.lockutils [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] Acquiring lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 792.970257] env[62736]: DEBUG oslo_concurrency.lockutils [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] Acquired lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 792.970415] env[62736]: DEBUG nova.network.neutron [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Refreshing network info cache for port e0c0a849-c8ce-4d35-a085-a6276922400b {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 792.970821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] Expecting reply to msg 63b9ae0450ff4db7866e0c209e585331 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 792.977834] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63b9ae0450ff4db7866e0c209e585331 [ 793.055699] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 793.057702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg ddf53e3ff0c04fc69cdc8504ff39e2e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 793.100110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ddf53e3ff0c04fc69cdc8504ff39e2e1 [ 793.116091] env[62736]: DEBUG nova.scheduler.client.report [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 793.118588] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 54e0bb1128834dacb37f3f95d558b986 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 793.138611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54e0bb1128834dacb37f3f95d558b986 [ 793.202443] env[62736]: ERROR nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 793.202443] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 793.202443] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 793.202443] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 793.202443] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 793.202443] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 793.202443] env[62736]: ERROR nova.compute.manager raise self.value [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 793.202443] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 793.202443] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 793.202443] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 793.202856] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 793.202856] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 793.202856] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 793.202856] env[62736]: ERROR nova.compute.manager [ 793.202856] env[62736]: Traceback (most recent call last): [ 793.202856] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 793.202856] env[62736]: listener.cb(fileno) [ 793.202856] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 793.202856] env[62736]: result = function(*args, **kwargs) [ 793.202856] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 793.202856] env[62736]: return func(*args, **kwargs) [ 793.202856] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 793.202856] env[62736]: raise e [ 793.202856] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 793.202856] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 793.202856] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 793.202856] env[62736]: created_port_ids = self._update_ports_for_instance( [ 793.202856] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 793.202856] env[62736]: with excutils.save_and_reraise_exception(): [ 793.202856] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 793.202856] env[62736]: self.force_reraise() [ 793.202856] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 793.202856] env[62736]: raise self.value [ 793.202856] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 793.202856] env[62736]: updated_port = self._update_port( [ 793.202856] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 793.202856] env[62736]: _ensure_no_port_binding_failure(port) [ 793.202856] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 793.202856] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 793.203606] env[62736]: nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 793.203606] env[62736]: Removing descriptor: 16 [ 793.253087] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 793.277456] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 793.277699] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 793.277856] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 793.278046] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 793.278165] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 793.278304] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 793.278499] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 793.278654] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 793.278813] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 793.278968] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 793.279133] env[62736]: DEBUG nova.virt.hardware [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 793.279975] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cbfa3ea-9eb2-4e25-9a53-ada7103e99ea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.287594] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d02b26e-50fe-4428-b281-6820cdded3c7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 793.301999] env[62736]: ERROR nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Traceback (most recent call last): [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] yield resources [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.driver.spawn(context, instance, image_meta, [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] vm_ref = self.build_virtual_machine(instance, [ 793.301999] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] vif_infos = vmwarevif.get_vif_info(self._session, [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] for vif in network_info: [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return self._sync_wrapper(fn, *args, **kwargs) [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.wait() [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self[:] = self._gt.wait() [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return self._exit_event.wait() [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 793.302398] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] current.throw(*self._exc) [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] result = function(*args, **kwargs) [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return func(*args, **kwargs) [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise e [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] nwinfo = self.network_api.allocate_for_instance( [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] created_port_ids = self._update_ports_for_instance( [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] with excutils.save_and_reraise_exception(): [ 793.302760] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.force_reraise() [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise self.value [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] updated_port = self._update_port( [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] _ensure_no_port_binding_failure(port) [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise exception.PortBindingFailed(port_id=port['id']) [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 793.303126] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] [ 793.303126] env[62736]: INFO nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Terminating instance [ 793.304312] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 793.495838] env[62736]: DEBUG nova.network.neutron [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 793.575618] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.620855] env[62736]: DEBUG nova.network.neutron [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 793.621381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] Expecting reply to msg fccdf8fc1de245b388f6d1c256427070 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 793.624688] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.411s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.625329] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 793.626952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 86d37cd37efc4c87875d1b24cc13bd59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 793.627896] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 25.974s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.629294] env[62736]: INFO nova.compute.claims [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 793.630790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 84e73ddc612243cd85a42dcf6be7ba4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 793.632066] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fccdf8fc1de245b388f6d1c256427070 [ 793.660630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86d37cd37efc4c87875d1b24cc13bd59 [ 793.664538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84e73ddc612243cd85a42dcf6be7ba4d [ 794.123714] env[62736]: DEBUG oslo_concurrency.lockutils [req-eb8c554c-1738-414b-8f8c-203ce25393bc req-1123fbbe-5a3a-4e7b-b384-adfe33ec6c7a service nova] Releasing lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 794.124215] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 794.124581] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 794.124909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 05abf662ce2749cda54bd7354bded2ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 794.131856] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05abf662ce2749cda54bd7354bded2ec [ 794.134285] env[62736]: DEBUG nova.compute.utils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 794.134856] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg ad95d1b3822f42a7ae5d8f7add4043a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 794.136996] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 1ed5a789ae094f509a1c3552e24fe349 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 794.138104] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 794.138271] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 794.147190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ed5a789ae094f509a1c3552e24fe349 [ 794.147611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad95d1b3822f42a7ae5d8f7add4043a6 [ 794.216351] env[62736]: DEBUG nova.policy [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '23c687bad35b409ab96bfd7cde54fd27', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9a349372836346d8983c95b50d980e37', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 794.619433] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Successfully created port: 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 794.641588] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 794.643387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 741191703c85432fa824fd060cf23f47 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 794.646241] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.679814] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 741191703c85432fa824fd060cf23f47 [ 794.745996] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 794.746469] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 4564a0010fcf406aa246f11cd117ea29 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 794.755431] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4564a0010fcf406aa246f11cd117ea29 [ 794.932933] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-887af572-0661-4e5b-ae37-7dc3f07d94fa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.941631] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cbbab2c-5be2-4192-a2c8-037bac783473 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.973687] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-890d5093-5357-4da1-bf06-0bbac7471186 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.981319] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50a27c83-7209-4c48-94de-b3c0b02fc884 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 794.997116] env[62736]: DEBUG nova.compute.provider_tree [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 794.997606] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 05148101359f4e4b8258fdd1c6a9c7f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.006513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05148101359f4e4b8258fdd1c6a9c7f0 [ 795.147795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 3cdfa21c44dc464890886708faf360c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.183583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3cdfa21c44dc464890886708faf360c6 [ 795.230824] env[62736]: DEBUG nova.compute.manager [req-5c1a3548-4054-41a7-8257-a88250e8660c req-d862898d-c810-48dc-b241-ed48c78869f9 service nova] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Received event network-vif-deleted-e0c0a849-c8ce-4d35-a085-a6276922400b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 795.248767] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 795.249113] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 795.249304] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 795.249922] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bac73f26-e6ea-48ed-ba0d-6ca51a7210ba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.258996] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6077711-c234-478b-9c29-38b2ff417b24 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.280346] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fd2547e6-9449-4839-8ebb-f02f01a85b8c could not be found. [ 795.280597] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 795.280781] env[62736]: INFO nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Took 0.03 seconds to destroy the instance on the hypervisor. [ 795.281021] env[62736]: DEBUG oslo.service.loopingcall [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 795.281237] env[62736]: DEBUG nova.compute.manager [-] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 795.281329] env[62736]: DEBUG nova.network.neutron [-] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 795.308204] env[62736]: DEBUG nova.network.neutron [-] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 795.308713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f0af23a31d7a4de2ad70646551899890 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.315921] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f0af23a31d7a4de2ad70646551899890 [ 795.484177] env[62736]: ERROR nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 795.484177] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.484177] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.484177] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.484177] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.484177] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.484177] env[62736]: ERROR nova.compute.manager raise self.value [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.484177] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 795.484177] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.484177] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 795.484600] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.484600] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 795.484600] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 795.484600] env[62736]: ERROR nova.compute.manager [ 795.484600] env[62736]: Traceback (most recent call last): [ 795.484600] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 795.484600] env[62736]: listener.cb(fileno) [ 795.484600] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 795.484600] env[62736]: result = function(*args, **kwargs) [ 795.484600] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 795.484600] env[62736]: return func(*args, **kwargs) [ 795.484600] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 795.484600] env[62736]: raise e [ 795.484600] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.484600] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 795.484600] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.484600] env[62736]: created_port_ids = self._update_ports_for_instance( [ 795.484600] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.484600] env[62736]: with excutils.save_and_reraise_exception(): [ 795.484600] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.484600] env[62736]: self.force_reraise() [ 795.484600] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.484600] env[62736]: raise self.value [ 795.484600] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.484600] env[62736]: updated_port = self._update_port( [ 795.484600] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.484600] env[62736]: _ensure_no_port_binding_failure(port) [ 795.484600] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.484600] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 795.485333] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 795.485333] env[62736]: Removing descriptor: 16 [ 795.500709] env[62736]: DEBUG nova.scheduler.client.report [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 795.502803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg a0bc9e97479a4ed4b70aeceb0e961ccc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.515719] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a0bc9e97479a4ed4b70aeceb0e961ccc [ 795.651462] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 795.672128] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 795.672367] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 795.672522] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 795.672698] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 795.672836] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 795.672974] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 795.673167] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 795.673317] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 795.673477] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 795.673632] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 795.673798] env[62736]: DEBUG nova.virt.hardware [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 795.674633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-326766a1-fd53-47fb-a93c-b7a924a423f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.682118] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-decd524e-1c34-4359-b87b-2bb600319865 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 795.695242] env[62736]: ERROR nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Traceback (most recent call last): [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] yield resources [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.driver.spawn(context, instance, image_meta, [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] vm_ref = self.build_virtual_machine(instance, [ 795.695242] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] vif_infos = vmwarevif.get_vif_info(self._session, [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] for vif in network_info: [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return self._sync_wrapper(fn, *args, **kwargs) [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.wait() [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self[:] = self._gt.wait() [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return self._exit_event.wait() [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 795.695526] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] current.throw(*self._exc) [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] result = function(*args, **kwargs) [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return func(*args, **kwargs) [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise e [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] nwinfo = self.network_api.allocate_for_instance( [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] created_port_ids = self._update_ports_for_instance( [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] with excutils.save_and_reraise_exception(): [ 795.695901] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.force_reraise() [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise self.value [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] updated_port = self._update_port( [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] _ensure_no_port_binding_failure(port) [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise exception.PortBindingFailed(port_id=port['id']) [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 795.696271] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] [ 795.696271] env[62736]: INFO nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Terminating instance [ 795.697568] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 795.697722] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquired lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 795.697884] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 795.698285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 7a6b4dff64dd4612843ef43b3169b147 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.706361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a6b4dff64dd4612843ef43b3169b147 [ 795.816820] env[62736]: DEBUG nova.network.neutron [-] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 795.817200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg adf2f907ab744fb88178386f1fb3254f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 795.828659] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adf2f907ab744fb88178386f1fb3254f [ 795.845149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "e2a1c5a7-34a1-4520-9b37-70f6087f0314" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 795.845376] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "e2a1c5a7-34a1-4520-9b37-70f6087f0314" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.008040] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.378s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.008040] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 796.008647] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 1e1ac710aace4b69b8b4bd699d6bf243 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.009668] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 25.613s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.013744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 65500bef9ef84c0cab6b1ef45d590bbe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.043648] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1e1ac710aace4b69b8b4bd699d6bf243 [ 796.044846] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65500bef9ef84c0cab6b1ef45d590bbe [ 796.214799] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.293904] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 796.294416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 071ce2803fe84aeda9db0deb24d1e4a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.302942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 071ce2803fe84aeda9db0deb24d1e4a1 [ 796.319340] env[62736]: INFO nova.compute.manager [-] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Took 1.04 seconds to deallocate network for instance. [ 796.321524] env[62736]: DEBUG nova.compute.claims [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 796.321691] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.519622] env[62736]: DEBUG nova.compute.utils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 796.520270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 3f6a66e4ddcc447abc4f1af0c865c947 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.521348] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 796.522060] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 796.532129] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f6a66e4ddcc447abc4f1af0c865c947 [ 796.565199] env[62736]: DEBUG nova.policy [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce05881c446e44ee9ca81e5416956ebb', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6362a2ff1b4046cb981b9552a7f621b1', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 796.776179] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61d13170-bd60-497b-922f-d6dbdc76c82e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.784234] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfd562fb-c68e-4233-80cd-c683bf7e6caf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.816421] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Releasing lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 796.816871] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 796.817057] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 796.817513] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9c4513c4-610a-4b85-a5d5-459325e76252 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.819891] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93d51212-e3b7-49cd-a76e-3fbccac50d09 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.832074] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-360c1255-45c6-482b-a2f9-f38298983708 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.842786] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-74317018-5a3b-4f76-a2bd-2c38ceee5544 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 796.846855] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Successfully created port: cc0c8455-7089-49be-8598-cb883d892f6d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 796.859346] env[62736]: DEBUG nova.compute.provider_tree [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 796.859870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg f7d35d4807ae420494868e48d4617d3f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.864598] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5014a870-32fd-4166-9446-c41ed2a4954e could not be found. [ 796.864852] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 796.865063] env[62736]: INFO nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Took 0.05 seconds to destroy the instance on the hypervisor. [ 796.865492] env[62736]: DEBUG oslo.service.loopingcall [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 796.865768] env[62736]: DEBUG nova.compute.manager [-] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 796.865908] env[62736]: DEBUG nova.network.neutron [-] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 796.867670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7d35d4807ae420494868e48d4617d3f [ 796.890764] env[62736]: DEBUG nova.network.neutron [-] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.891227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ed71ef633e514d05b8babe78b66851fb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 796.897930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed71ef633e514d05b8babe78b66851fb [ 797.025078] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 797.026806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg a5e499b6f75c4f7c8091a49e909b85aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.058099] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5e499b6f75c4f7c8091a49e909b85aa [ 797.259731] env[62736]: DEBUG nova.compute.manager [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Received event network-changed-30f94bf9-c081-4c9e-8c94-4a50ed6b7af0 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 797.259995] env[62736]: DEBUG nova.compute.manager [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Refreshing instance network info cache due to event network-changed-30f94bf9-c081-4c9e-8c94-4a50ed6b7af0. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 797.260143] env[62736]: DEBUG oslo_concurrency.lockutils [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] Acquiring lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 797.260284] env[62736]: DEBUG oslo_concurrency.lockutils [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] Acquired lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 797.260462] env[62736]: DEBUG nova.network.neutron [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Refreshing network info cache for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 797.260875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] Expecting reply to msg c99fcbce765a412cb753e93155bac66c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.267860] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c99fcbce765a412cb753e93155bac66c [ 797.367330] env[62736]: DEBUG nova.scheduler.client.report [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 797.369713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 7748f56be621489aae23af979dc780b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.381328] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7748f56be621489aae23af979dc780b7 [ 797.393259] env[62736]: DEBUG nova.network.neutron [-] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.393927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ef9f80152a4746c38f78b001d2e4a4b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.402956] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef9f80152a4746c38f78b001d2e4a4b4 [ 797.532592] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 269f176caac2478e88591059cff07975 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.565140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 269f176caac2478e88591059cff07975 [ 797.706115] env[62736]: ERROR nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 797.706115] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.706115] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.706115] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.706115] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.706115] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.706115] env[62736]: ERROR nova.compute.manager raise self.value [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.706115] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 797.706115] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.706115] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 797.706518] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.706518] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 797.706518] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 797.706518] env[62736]: ERROR nova.compute.manager [ 797.706518] env[62736]: Traceback (most recent call last): [ 797.706518] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 797.706518] env[62736]: listener.cb(fileno) [ 797.706518] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 797.706518] env[62736]: result = function(*args, **kwargs) [ 797.706518] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 797.706518] env[62736]: return func(*args, **kwargs) [ 797.706518] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 797.706518] env[62736]: raise e [ 797.706518] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.706518] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 797.706518] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.706518] env[62736]: created_port_ids = self._update_ports_for_instance( [ 797.706518] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.706518] env[62736]: with excutils.save_and_reraise_exception(): [ 797.706518] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.706518] env[62736]: self.force_reraise() [ 797.706518] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.706518] env[62736]: raise self.value [ 797.706518] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.706518] env[62736]: updated_port = self._update_port( [ 797.706518] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.706518] env[62736]: _ensure_no_port_binding_failure(port) [ 797.706518] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.706518] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 797.707150] env[62736]: nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 797.707150] env[62736]: Removing descriptor: 16 [ 797.777080] env[62736]: DEBUG nova.network.neutron [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 797.856325] env[62736]: DEBUG nova.network.neutron [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.857132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] Expecting reply to msg ed313a46e12c4d96942d3628e4ed12bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.866140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed313a46e12c4d96942d3628e4ed12bd [ 797.871664] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.862s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.872279] env[62736]: ERROR nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Traceback (most recent call last): [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.driver.spawn(context, instance, image_meta, [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] vm_ref = self.build_virtual_machine(instance, [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] vif_infos = vmwarevif.get_vif_info(self._session, [ 797.872279] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] for vif in network_info: [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return self._sync_wrapper(fn, *args, **kwargs) [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.wait() [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self[:] = self._gt.wait() [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return self._exit_event.wait() [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] current.throw(*self._exc) [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 797.872585] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] result = function(*args, **kwargs) [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] return func(*args, **kwargs) [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise e [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] nwinfo = self.network_api.allocate_for_instance( [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] created_port_ids = self._update_ports_for_instance( [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] with excutils.save_and_reraise_exception(): [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] self.force_reraise() [ 797.872878] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise self.value [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] updated_port = self._update_port( [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] _ensure_no_port_binding_failure(port) [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] raise exception.PortBindingFailed(port_id=port['id']) [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] nova.exception.PortBindingFailed: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. [ 797.873168] env[62736]: ERROR nova.compute.manager [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] [ 797.873168] env[62736]: DEBUG nova.compute.utils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 797.874097] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 26.425s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.874271] env[62736]: DEBUG nova.objects.instance [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62736) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 797.875715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 98f534b03295433da352455ba74e87fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.877126] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Build of instance 3135faf8-b918-42f5-bcaa-c23a19166bb3 was re-scheduled: Binding failed for port 7546ffee-78bf-462a-a94a-d087f5f1b40d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 797.877545] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 797.877765] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquiring lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 797.877904] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Acquired lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 797.878058] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 797.878412] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 99f15eea06a5408099ae7d30122d9cc7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 797.891579] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99f15eea06a5408099ae7d30122d9cc7 [ 797.895605] env[62736]: INFO nova.compute.manager [-] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Took 1.03 seconds to deallocate network for instance. [ 797.897965] env[62736]: DEBUG nova.compute.claims [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 797.898187] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.912180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98f534b03295433da352455ba74e87fd [ 798.036625] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 798.062272] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 798.062529] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 798.062678] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 798.062845] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 798.062981] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 798.063120] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 798.063314] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 798.063460] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 798.063608] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 798.063758] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 798.063916] env[62736]: DEBUG nova.virt.hardware [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 798.064788] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff7e4b98-2dfb-4b8c-aaaf-f058947952f8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.074078] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cbcb2b1-46e7-4387-a5fb-31834b637567 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 798.086944] env[62736]: ERROR nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Traceback (most recent call last): [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] yield resources [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.driver.spawn(context, instance, image_meta, [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] vm_ref = self.build_virtual_machine(instance, [ 798.086944] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] for vif in network_info: [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return self._sync_wrapper(fn, *args, **kwargs) [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.wait() [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self[:] = self._gt.wait() [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return self._exit_event.wait() [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 798.087255] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] current.throw(*self._exc) [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] result = function(*args, **kwargs) [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return func(*args, **kwargs) [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise e [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] nwinfo = self.network_api.allocate_for_instance( [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] created_port_ids = self._update_ports_for_instance( [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] with excutils.save_and_reraise_exception(): [ 798.087569] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.force_reraise() [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise self.value [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] updated_port = self._update_port( [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] _ensure_no_port_binding_failure(port) [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise exception.PortBindingFailed(port_id=port['id']) [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 798.088192] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] [ 798.088192] env[62736]: INFO nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Terminating instance [ 798.089315] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquiring lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 798.089492] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquired lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 798.089665] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 798.090072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 37697a5e6e9b428ebc9bff83bad974f5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.098428] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37697a5e6e9b428ebc9bff83bad974f5 [ 798.359684] env[62736]: DEBUG oslo_concurrency.lockutils [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] Releasing lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 798.359979] env[62736]: DEBUG nova.compute.manager [req-58533f42-1caa-4122-b6d9-c51f73034aa9 req-8ae43f8e-ec50-4740-bc2b-a19f634661ed service nova] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Received event network-vif-deleted-30f94bf9-c081-4c9e-8c94-4a50ed6b7af0 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 798.379541] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg f34d914b628941dc98bd2c36c22d55e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.386056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f34d914b628941dc98bd2c36c22d55e6 [ 798.401599] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.473570] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.474162] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg ad281d83924a4ad1ba60341a0463623e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.482371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad281d83924a4ad1ba60341a0463623e [ 798.608407] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.696875] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 798.697429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 98f3dafec9994103912befacd4c11eea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.711792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98f3dafec9994103912befacd4c11eea [ 798.883367] env[62736]: DEBUG oslo_concurrency.lockutils [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.009s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 798.883694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-69a8d882-d36f-4a31-a056-3d9fb89290dd tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg e6873bd0bd094561a184097ed4432f58 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.884649] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.794s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.886186] env[62736]: INFO nova.compute.claims [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 798.888539] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 2227c6d5ba5846c196ce611b29ccac17 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.899458] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6873bd0bd094561a184097ed4432f58 [ 798.925851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2227c6d5ba5846c196ce611b29ccac17 [ 798.976606] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Releasing lock "refresh_cache-3135faf8-b918-42f5-bcaa-c23a19166bb3" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 798.976826] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 798.977006] env[62736]: DEBUG nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 798.977174] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 798.992665] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 798.993279] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg b7a9ff8904b04ab78a6538731d35f82e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 798.999982] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b7a9ff8904b04ab78a6538731d35f82e [ 799.205828] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Releasing lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 799.206284] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 799.206473] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 799.206822] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-83b77d02-7653-4b8a-887c-9ac5687fbf68 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.215694] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee6849f8-c4b8-40c2-816f-273eee148342 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 799.236651] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b49cde53-0ef7-44a4-b894-e37d1cbf41a7 could not be found. [ 799.236954] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 799.237042] env[62736]: INFO nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Took 0.03 seconds to destroy the instance on the hypervisor. [ 799.237271] env[62736]: DEBUG oslo.service.loopingcall [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 799.237623] env[62736]: DEBUG nova.compute.manager [-] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 799.237719] env[62736]: DEBUG nova.network.neutron [-] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 799.252012] env[62736]: DEBUG nova.network.neutron [-] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 799.252503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6978157a677245e2aacb51e1edd29288 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.264931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6978157a677245e2aacb51e1edd29288 [ 799.290805] env[62736]: DEBUG nova.compute.manager [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Received event network-changed-cc0c8455-7089-49be-8598-cb883d892f6d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 799.290981] env[62736]: DEBUG nova.compute.manager [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Refreshing instance network info cache due to event network-changed-cc0c8455-7089-49be-8598-cb883d892f6d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 799.291202] env[62736]: DEBUG oslo_concurrency.lockutils [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] Acquiring lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 799.291341] env[62736]: DEBUG oslo_concurrency.lockutils [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] Acquired lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 799.291492] env[62736]: DEBUG nova.network.neutron [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Refreshing network info cache for port cc0c8455-7089-49be-8598-cb883d892f6d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 799.291944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] Expecting reply to msg 7944390bbabc41a082f77d188751120e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.298274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7944390bbabc41a082f77d188751120e [ 799.391743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 1bab97c9a6de472e92135f6097d95ffa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.399679] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bab97c9a6de472e92135f6097d95ffa [ 799.495608] env[62736]: DEBUG nova.network.neutron [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.496143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 197eaad355d144699f34246b1906059f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.504074] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 197eaad355d144699f34246b1906059f [ 799.759832] env[62736]: DEBUG nova.network.neutron [-] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.760321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b277652a7ece49b996299333eaea23d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.770144] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b277652a7ece49b996299333eaea23d0 [ 799.820755] env[62736]: DEBUG nova.network.neutron [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 799.892787] env[62736]: DEBUG nova.network.neutron [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 799.893283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] Expecting reply to msg 503869efff2f4be6ac707ef9fbeae490 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 799.902980] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 503869efff2f4be6ac707ef9fbeae490 [ 799.998708] env[62736]: INFO nova.compute.manager [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] [instance: 3135faf8-b918-42f5-bcaa-c23a19166bb3] Took 1.02 seconds to deallocate network for instance. [ 800.000410] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 29aa96c37b4442a2b8207841e8c44c42 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 800.037145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 29aa96c37b4442a2b8207841e8c44c42 [ 800.145485] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5327e573-0e1d-4287-acc5-dbe6c3e21b6d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.154681] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d312008-26b7-4afa-b040-b6225ee0599b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.183502] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0334821e-5109-47da-956a-c5bb9f6bc1f6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.190239] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bf3073c-2ebc-45bc-b8a0-f2e524cbe4de {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 800.202724] env[62736]: DEBUG nova.compute.provider_tree [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 800.203204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 4b8579be59c641c892d2896273d8f6a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 800.210616] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b8579be59c641c892d2896273d8f6a3 [ 800.265743] env[62736]: INFO nova.compute.manager [-] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Took 1.03 seconds to deallocate network for instance. [ 800.268028] env[62736]: DEBUG nova.compute.claims [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 800.268200] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.395518] env[62736]: DEBUG oslo_concurrency.lockutils [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] Releasing lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 800.395861] env[62736]: DEBUG nova.compute.manager [req-e5937024-87c2-4199-8c77-4d808d6a84dc req-07d20f26-db85-4255-b181-b1b1e18995a9 service nova] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Received event network-vif-deleted-cc0c8455-7089-49be-8598-cb883d892f6d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 800.514535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 6ac745c4b6f14c54b0785459b9c8fd99 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 800.544124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ac745c4b6f14c54b0785459b9c8fd99 [ 800.705793] env[62736]: DEBUG nova.scheduler.client.report [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 800.708252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 4718bafecb644d63b6be18819d07fe55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 800.719813] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4718bafecb644d63b6be18819d07fe55 [ 801.034014] env[62736]: INFO nova.scheduler.client.report [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Deleted allocations for instance 3135faf8-b918-42f5-bcaa-c23a19166bb3 [ 801.039999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Expecting reply to msg 1551fd7e42794ce1a36b35b57465e06d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 801.049546] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1551fd7e42794ce1a36b35b57465e06d [ 801.210686] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.326s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 801.211220] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 801.212974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 06fdca674aee458192e6dd5aff8f6f78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 801.214300] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.559s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 801.215916] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 855bee0d6e3d4a3cb81ba9221b863d25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 801.247704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 855bee0d6e3d4a3cb81ba9221b863d25 [ 801.249630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06fdca674aee458192e6dd5aff8f6f78 [ 801.541851] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ea32325d-dfbc-429e-8784-2a6cf8c1f4da tempest-ServersNegativeTestMultiTenantJSON-2037149078 tempest-ServersNegativeTestMultiTenantJSON-2037149078-project-member] Lock "3135faf8-b918-42f5-bcaa-c23a19166bb3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 157.436s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 801.542463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg d23643ad08a34000ad9b4f2d219be241 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 801.552776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d23643ad08a34000ad9b4f2d219be241 [ 801.719452] env[62736]: DEBUG nova.compute.utils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 801.720109] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 904f390c8c5747b29f11347fd73f888c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 801.726326] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 801.726504] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 801.735489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 904f390c8c5747b29f11347fd73f888c [ 801.771608] env[62736]: DEBUG nova.policy [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6a6d2a917fec48e48e943137dff043fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd781d0a045444391bc070520be1b6431', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 801.989535] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a869145-609c-4215-9f23-3e0fab1995a3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 801.997153] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc8fc132-a0e5-4e7b-aa77-26e4bbacacd8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.026649] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f021ec96-b473-42ff-afd5-c7ed240acd78 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.033605] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-573b9abb-836c-43b6-984a-695a251bbaa0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 802.052079] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 802.052079] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg abd8ae1ade874046baa8ad7a3d61ca04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 802.052079] env[62736]: DEBUG nova.compute.provider_tree [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 802.052079] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 13f98c0b4c624cb8a6f3b9524b075e25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 802.066205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13f98c0b4c624cb8a6f3b9524b075e25 [ 802.083913] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Successfully created port: 77e7b973-54f1-4090-9007-9bdb448043dd {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 802.086529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abd8ae1ade874046baa8ad7a3d61ca04 [ 802.227730] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 802.229648] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 3570b717707d4bac82b304b15f87edd6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 802.262196] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3570b717707d4bac82b304b15f87edd6 [ 802.554471] env[62736]: DEBUG nova.scheduler.client.report [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 802.556816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 9c57c3e7add84ade8b633f29990e66e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 802.569484] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.595615] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c57c3e7add84ade8b633f29990e66e0 [ 802.734702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 2a92d6fdecfa492e890cdec4b4dfd65c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 802.765703] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a92d6fdecfa492e890cdec4b4dfd65c [ 803.029888] env[62736]: DEBUG nova.compute.manager [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Received event network-changed-77e7b973-54f1-4090-9007-9bdb448043dd {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 803.030083] env[62736]: DEBUG nova.compute.manager [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Refreshing instance network info cache due to event network-changed-77e7b973-54f1-4090-9007-9bdb448043dd. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 803.030317] env[62736]: DEBUG oslo_concurrency.lockutils [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] Acquiring lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.030429] env[62736]: DEBUG oslo_concurrency.lockutils [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] Acquired lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 803.030920] env[62736]: DEBUG nova.network.neutron [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Refreshing network info cache for port 77e7b973-54f1-4090-9007-9bdb448043dd {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 803.030998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] Expecting reply to msg 590a1ff27148443aba6e8d66a88233cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.041098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 590a1ff27148443aba6e8d66a88233cf [ 803.052093] env[62736]: ERROR nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 803.052093] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.052093] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 803.052093] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 803.052093] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.052093] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.052093] env[62736]: ERROR nova.compute.manager raise self.value [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 803.052093] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 803.052093] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.052093] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 803.052497] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.052497] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 803.052497] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 803.052497] env[62736]: ERROR nova.compute.manager [ 803.052497] env[62736]: Traceback (most recent call last): [ 803.052497] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 803.052497] env[62736]: listener.cb(fileno) [ 803.052497] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 803.052497] env[62736]: result = function(*args, **kwargs) [ 803.052497] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 803.052497] env[62736]: return func(*args, **kwargs) [ 803.052497] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 803.052497] env[62736]: raise e [ 803.052497] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.052497] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 803.052497] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 803.052497] env[62736]: created_port_ids = self._update_ports_for_instance( [ 803.052497] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 803.052497] env[62736]: with excutils.save_and_reraise_exception(): [ 803.052497] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.052497] env[62736]: self.force_reraise() [ 803.052497] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.052497] env[62736]: raise self.value [ 803.052497] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 803.052497] env[62736]: updated_port = self._update_port( [ 803.052497] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.052497] env[62736]: _ensure_no_port_binding_failure(port) [ 803.052497] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.052497] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 803.053189] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 803.053189] env[62736]: Removing descriptor: 16 [ 803.060537] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.845s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 803.060537] env[62736]: ERROR nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Traceback (most recent call last): [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.driver.spawn(context, instance, image_meta, [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 803.060537] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] vm_ref = self.build_virtual_machine(instance, [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] vif_infos = vmwarevif.get_vif_info(self._session, [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] for vif in network_info: [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return self._sync_wrapper(fn, *args, **kwargs) [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.wait() [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self[:] = self._gt.wait() [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return self._exit_event.wait() [ 803.060774] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] current.throw(*self._exc) [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] result = function(*args, **kwargs) [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] return func(*args, **kwargs) [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise e [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] nwinfo = self.network_api.allocate_for_instance( [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] created_port_ids = self._update_ports_for_instance( [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 803.061063] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] with excutils.save_and_reraise_exception(): [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] self.force_reraise() [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise self.value [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] updated_port = self._update_port( [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] _ensure_no_port_binding_failure(port) [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] raise exception.PortBindingFailed(port_id=port['id']) [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] nova.exception.PortBindingFailed: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. [ 803.061348] env[62736]: ERROR nova.compute.manager [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] [ 803.061626] env[62736]: DEBUG nova.compute.utils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 803.062697] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 26.702s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 803.064907] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 42bbec8a97564f5ab8834bc61f86fcaf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.069409] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Build of instance 97927074-a676-4ebd-aa7a-fe92e3ce1be6 was re-scheduled: Binding failed for port 95eb6170-9245-415c-9bdb-5a5437c7ddb8, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 803.069409] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 803.069409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.069409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquired lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 803.069776] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 803.069776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 927b909ff22747429454496cdd3b7215 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.077046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 927b909ff22747429454496cdd3b7215 [ 803.118561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42bbec8a97564f5ab8834bc61f86fcaf [ 803.237432] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 803.261484] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 803.261737] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 803.261890] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 803.262069] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 803.262208] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 803.262752] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 803.262752] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 803.262752] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 803.263044] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 803.263044] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 803.263381] env[62736]: DEBUG nova.virt.hardware [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 803.264095] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-134b2a01-b70b-4932-b731-d57945a00c75 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.272100] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29f65c13-ba82-4070-a21b-6585dc89032d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.285667] env[62736]: ERROR nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Traceback (most recent call last): [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] yield resources [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.driver.spawn(context, instance, image_meta, [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] vm_ref = self.build_virtual_machine(instance, [ 803.285667] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] vif_infos = vmwarevif.get_vif_info(self._session, [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] for vif in network_info: [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return self._sync_wrapper(fn, *args, **kwargs) [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.wait() [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self[:] = self._gt.wait() [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return self._exit_event.wait() [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 803.286006] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] current.throw(*self._exc) [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] result = function(*args, **kwargs) [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return func(*args, **kwargs) [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise e [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] nwinfo = self.network_api.allocate_for_instance( [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] created_port_ids = self._update_ports_for_instance( [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] with excutils.save_and_reraise_exception(): [ 803.286399] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.force_reraise() [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise self.value [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] updated_port = self._update_port( [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] _ensure_no_port_binding_failure(port) [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise exception.PortBindingFailed(port_id=port['id']) [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 803.286728] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] [ 803.286728] env[62736]: INFO nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Terminating instance [ 803.287929] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 803.585408] env[62736]: DEBUG nova.network.neutron [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.595011] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 803.655709] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.656253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg a94edde00fed47c4b33f01762be61c0b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.666530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a94edde00fed47c4b33f01762be61c0b [ 803.703703] env[62736]: DEBUG nova.network.neutron [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 803.704202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] Expecting reply to msg dbc85e48375f472aafd198dacff3bc1f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.711992] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbc85e48375f472aafd198dacff3bc1f [ 803.849182] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e6c3bb73-e7df-4407-903e-09341b6a42f4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.856737] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ea22fc1-2e19-4dac-84bf-c21f405b6204 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.889198] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-10f48f28-a8ff-4a2f-ae86-02582314b759 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.896154] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e05dcdc-cbde-4df6-9506-d06b20f31e5f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 803.911181] env[62736]: DEBUG nova.compute.provider_tree [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 803.911824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg c4c3105bdf7b431392fed8f2a76fa57c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 803.921829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4c3105bdf7b431392fed8f2a76fa57c [ 804.159047] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Releasing lock "refresh_cache-97927074-a676-4ebd-aa7a-fe92e3ce1be6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 804.159321] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 804.159508] env[62736]: DEBUG nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 804.159685] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 804.173722] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.174552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 0440b79fda7d4ef48f22db7a33731e5d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.181340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0440b79fda7d4ef48f22db7a33731e5d [ 804.206385] env[62736]: DEBUG oslo_concurrency.lockutils [req-65ca11a3-29f8-4493-8bb7-4464d65da097 req-aa594a0a-25cc-4b7e-8265-3e1d3553e25b service nova] Releasing lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 804.206758] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 804.206932] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 804.207530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg c6de1235247945e3a813b7132df9c5d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.214095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6de1235247945e3a813b7132df9c5d5 [ 804.415090] env[62736]: DEBUG nova.scheduler.client.report [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 804.417597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 1b6bef85f87d4439be335377f1461a89 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.434505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b6bef85f87d4439be335377f1461a89 [ 804.676580] env[62736]: DEBUG nova.network.neutron [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.677122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 1fd430e956fc407cbcc9fe965dbf1818 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.684841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1fd430e956fc407cbcc9fe965dbf1818 [ 804.722486] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.797516] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.798048] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 0572f3b10b5947d0be7154844d0e22f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.813324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0572f3b10b5947d0be7154844d0e22f1 [ 804.920796] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.858s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 804.921433] env[62736]: ERROR nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Traceback (most recent call last): [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.driver.spawn(context, instance, image_meta, [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] vm_ref = self.build_virtual_machine(instance, [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] vif_infos = vmwarevif.get_vif_info(self._session, [ 804.921433] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] for vif in network_info: [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return self._sync_wrapper(fn, *args, **kwargs) [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.wait() [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self[:] = self._gt.wait() [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return self._exit_event.wait() [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] current.throw(*self._exc) [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 804.921742] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] result = function(*args, **kwargs) [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] return func(*args, **kwargs) [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise e [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] nwinfo = self.network_api.allocate_for_instance( [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] created_port_ids = self._update_ports_for_instance( [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] with excutils.save_and_reraise_exception(): [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] self.force_reraise() [ 804.922028] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise self.value [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] updated_port = self._update_port( [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] _ensure_no_port_binding_failure(port) [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] raise exception.PortBindingFailed(port_id=port['id']) [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] nova.exception.PortBindingFailed: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. [ 804.922318] env[62736]: ERROR nova.compute.manager [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] [ 804.922318] env[62736]: DEBUG nova.compute.utils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 804.923408] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 26.164s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 804.925028] env[62736]: INFO nova.compute.claims [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 804.926596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 08e67488de1940e9b341296d9a5140fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.927938] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Build of instance 61f31838-2fd1-4804-82b7-56b3a08c3ff5 was re-scheduled: Binding failed for port 993db6d2-a560-4b87-9cd6-1bd027525b39, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 804.928371] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 804.928596] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquiring lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 804.928832] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Acquired lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 804.928996] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 804.929428] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 966ac523124846caa3861970dcaf28a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 804.936205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 966ac523124846caa3861970dcaf28a6 [ 804.966630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08e67488de1940e9b341296d9a5140fd [ 805.114266] env[62736]: DEBUG nova.compute.manager [req-1facbefb-aa34-4d74-b0d0-65534a9075e8 req-cf7b5daf-3a8e-4233-96b2-d38f06a2f7a4 service nova] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Received event network-vif-deleted-77e7b973-54f1-4090-9007-9bdb448043dd {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 805.179018] env[62736]: INFO nova.compute.manager [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 97927074-a676-4ebd-aa7a-fe92e3ce1be6] Took 1.02 seconds to deallocate network for instance. [ 805.180736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 6a0289e150714aeaa98e74cd2fe28aa2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.227743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a0289e150714aeaa98e74cd2fe28aa2 [ 805.300621] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 805.301069] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 805.301255] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 805.301544] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c74f33b0-a3ea-427a-93cf-2864c398fcb0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.311931] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-364cc7b1-c116-493c-837c-0b80ae4ad9b3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 805.336337] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 25e906f6-ec98-4c99-83c7-43f94c5790dd could not be found. [ 805.336554] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 805.336734] env[62736]: INFO nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Took 0.04 seconds to destroy the instance on the hypervisor. [ 805.336968] env[62736]: DEBUG oslo.service.loopingcall [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 805.337173] env[62736]: DEBUG nova.compute.manager [-] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 805.337265] env[62736]: DEBUG nova.network.neutron [-] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 805.429972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg bf8a672563724e5a8a1cc6eea39d0e80 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.444086] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf8a672563724e5a8a1cc6eea39d0e80 [ 805.445086] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 805.474166] env[62736]: DEBUG nova.network.neutron [-] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 805.474678] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1b2396d69a924dce90801cb09bbbe1d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.484466] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b2396d69a924dce90801cb09bbbe1d0 [ 805.499308] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.499865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 171b891f145e4cfaa1d570984da2c7f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.507625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 171b891f145e4cfaa1d570984da2c7f7 [ 805.685055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 84725126551f41c595573d3c532b7a97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.738181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84725126551f41c595573d3c532b7a97 [ 805.976629] env[62736]: DEBUG nova.network.neutron [-] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 805.977069] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6b74b53cf4f745c08e047ef9a7396f7c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 805.985916] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b74b53cf4f745c08e047ef9a7396f7c [ 806.001713] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Releasing lock "refresh_cache-61f31838-2fd1-4804-82b7-56b3a08c3ff5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 806.001899] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 806.002059] env[62736]: DEBUG nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 806.002217] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 806.018730] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 806.019320] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 5c05c138af184c3a87c049cd5883a04a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.026116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c05c138af184c3a87c049cd5883a04a [ 806.191866] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8e52db6-0bbb-4902-b45a-7fc52ee5ae0d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.199150] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4924f66-d335-46e6-a5c0-ac2d2a2bde5b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.229631] env[62736]: INFO nova.scheduler.client.report [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Deleted allocations for instance 97927074-a676-4ebd-aa7a-fe92e3ce1be6 [ 806.243065] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98457e3d-5a15-48f6-a760-277949959457 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.253452] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 4b91c4ebca9144dd9758913423ae48ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.261216] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85b65d3d-15a5-4347-b084-6d6a99de022a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 806.266701] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b91c4ebca9144dd9758913423ae48ce [ 806.276135] env[62736]: DEBUG nova.compute.provider_tree [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 806.276135] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 5293058643b742c1a823409c5492d947 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.281473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5293058643b742c1a823409c5492d947 [ 806.481202] env[62736]: INFO nova.compute.manager [-] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Took 1.14 seconds to deallocate network for instance. [ 806.482246] env[62736]: DEBUG nova.compute.claims [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 806.482246] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.523055] env[62736]: DEBUG nova.network.neutron [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 806.523055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 957987399e1d45fcaf3fc2ac9bb5d9ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.531392] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 957987399e1d45fcaf3fc2ac9bb5d9ae [ 806.758299] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9640061f-8bc3-474f-9486-8a68eaa19a56 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "97927074-a676-4ebd-aa7a-fe92e3ce1be6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 162.315s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.758299] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg ae81e1ce68194878a0bba7c7d835ae4e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.767002] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae81e1ce68194878a0bba7c7d835ae4e [ 806.778699] env[62736]: DEBUG nova.scheduler.client.report [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 806.781187] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 4e6f016965ae46aa8c753098888d8a3c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 806.792772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e6f016965ae46aa8c753098888d8a3c [ 807.025307] env[62736]: INFO nova.compute.manager [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] [instance: 61f31838-2fd1-4804-82b7-56b3a08c3ff5] Took 1.02 seconds to deallocate network for instance. [ 807.027336] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg b2dbde492b664ecc800902eea4909e62 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.057999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2dbde492b664ecc800902eea4909e62 [ 807.260572] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 807.262230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg baa73103b9b54c77b0d16f050c1f0263 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.283253] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.360s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 807.283754] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 807.285361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg bee3738751e74ba5890a77f9dd4d2356 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.286361] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 27.127s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 807.286539] env[62736]: DEBUG nova.objects.instance [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] [instance: bbc25649-cfa4-4f5f-ab1e-5f1347780612] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62736) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 807.287927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg 61f79b9a6f9a4266b4687a5d90c8a85d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.290860] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg baa73103b9b54c77b0d16f050c1f0263 [ 807.315478] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61f79b9a6f9a4266b4687a5d90c8a85d [ 807.316045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bee3738751e74ba5890a77f9dd4d2356 [ 807.532037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg b8e6f301a9f84ab88232db3a3e4a3543 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.561284] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8e6f301a9f84ab88232db3a3e4a3543 [ 807.778973] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 807.791463] env[62736]: DEBUG nova.compute.utils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 807.792171] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 059cfa0d92384c4a85d29a41126ba2f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.795108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg 4d440b948eeb47cc9b0c5a0e3094d5c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 807.796322] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 807.796322] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 807.803999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 059cfa0d92384c4a85d29a41126ba2f0 [ 807.806506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d440b948eeb47cc9b0c5a0e3094d5c1 [ 807.857059] env[62736]: DEBUG nova.policy [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '303068b2037f422b9cba37aebaca71ab', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9703f47d3d944740a48535c270de37ed', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 808.052185] env[62736]: INFO nova.scheduler.client.report [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Deleted allocations for instance 61f31838-2fd1-4804-82b7-56b3a08c3ff5 [ 808.057696] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Expecting reply to msg 14427f584bf84d9bac2d7b29ba96ecb6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.077008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14427f584bf84d9bac2d7b29ba96ecb6 [ 808.106197] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Successfully created port: 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 808.296710] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 808.298398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg f3a62f00605a4b56b2ed914d38080958 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.300177] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.014s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 808.300556] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b9ef691f-78d7-4ed5-a8af-6b87611f17aa tempest-ServersAdmin275Test-881461725 tempest-ServersAdmin275Test-881461725-project-admin] Expecting reply to msg e0df0edb413a4d7bb09cb915c53772de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.301432] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 23.387s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 808.301680] env[62736]: DEBUG nova.objects.instance [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lazy-loading 'resources' on Instance uuid bbc25649-cfa4-4f5f-ab1e-5f1347780612 {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 808.301912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg f01bf9dd9c5845c996fd64c1ace90d42 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.313620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0df0edb413a4d7bb09cb915c53772de [ 808.328296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f01bf9dd9c5845c996fd64c1ace90d42 [ 808.342564] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3a62f00605a4b56b2ed914d38080958 [ 808.559298] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e1534752-5daa-4e5a-9b8b-021679612a65 tempest-ListImageFiltersTestJSON-1393988037 tempest-ListImageFiltersTestJSON-1393988037-project-member] Lock "61f31838-2fd1-4804-82b7-56b3a08c3ff5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 163.641s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 808.560068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg ae6c4ba1e67845018ada1bd4daa2938e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.569478] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae6c4ba1e67845018ada1bd4daa2938e [ 808.805585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 63fc2e5d57194b7f8882895d3f6be53b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.847050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63fc2e5d57194b7f8882895d3f6be53b [ 808.943758] env[62736]: DEBUG nova.compute.manager [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Received event network-changed-6696136d-efd5-48c3-84b6-c8fcd9d3aa8a {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 808.943972] env[62736]: DEBUG nova.compute.manager [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Refreshing instance network info cache due to event network-changed-6696136d-efd5-48c3-84b6-c8fcd9d3aa8a. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 808.944197] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] Acquiring lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 808.944337] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] Acquired lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 808.944511] env[62736]: DEBUG nova.network.neutron [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Refreshing network info cache for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 808.944924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] Expecting reply to msg 4232c8c3c6e54b919f8f8e0d1d80761d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 808.951874] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4232c8c3c6e54b919f8f8e0d1d80761d [ 809.062649] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 809.064339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 199b9c0335be444dbe122c27f37955e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 809.065820] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b334b83f-cccd-41ae-9f38-fa73b2e745be {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.073833] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7977d89f-f1ed-4cfd-aecb-f8d6080cb73a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.105282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 199b9c0335be444dbe122c27f37955e1 [ 809.106364] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d430e90a-d0f9-426d-9ab1-3288109c6592 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.115216] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c1d85e6-1c7a-42d5-88ee-14304962e8f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.130738] env[62736]: DEBUG nova.compute.provider_tree [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 809.131191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 1916e4d5ffbb47aabba1f21bdb320afe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 809.138263] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1916e4d5ffbb47aabba1f21bdb320afe [ 809.155220] env[62736]: ERROR nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 809.155220] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.155220] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.155220] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.155220] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.155220] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.155220] env[62736]: ERROR nova.compute.manager raise self.value [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.155220] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 809.155220] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.155220] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 809.155671] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.155671] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 809.155671] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 809.155671] env[62736]: ERROR nova.compute.manager [ 809.156210] env[62736]: Traceback (most recent call last): [ 809.156263] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 809.156263] env[62736]: listener.cb(fileno) [ 809.156263] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.156263] env[62736]: result = function(*args, **kwargs) [ 809.156263] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.156263] env[62736]: return func(*args, **kwargs) [ 809.156263] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.156263] env[62736]: raise e [ 809.156263] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.156263] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 809.156263] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.156263] env[62736]: created_port_ids = self._update_ports_for_instance( [ 809.156263] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.156263] env[62736]: with excutils.save_and_reraise_exception(): [ 809.156263] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.156263] env[62736]: self.force_reraise() [ 809.156263] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.156263] env[62736]: raise self.value [ 809.156263] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.156263] env[62736]: updated_port = self._update_port( [ 809.156263] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.156263] env[62736]: _ensure_no_port_binding_failure(port) [ 809.156263] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.156263] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 809.156263] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 809.157020] env[62736]: Removing descriptor: 17 [ 809.311500] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 809.337531] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 809.337720] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 809.337876] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 809.338054] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 809.338195] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 809.338337] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 809.338536] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 809.338702] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 809.338848] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 809.339002] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 809.339164] env[62736]: DEBUG nova.virt.hardware [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 809.340052] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a3f06aa-11cd-4d4a-bcc9-8f8a4ccfa6ad {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.347979] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65386d0d-f36a-4915-898e-af99cef16645 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 809.362541] env[62736]: ERROR nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] Traceback (most recent call last): [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] yield resources [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.driver.spawn(context, instance, image_meta, [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] vm_ref = self.build_virtual_machine(instance, [ 809.362541] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] vif_infos = vmwarevif.get_vif_info(self._session, [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] for vif in network_info: [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return self._sync_wrapper(fn, *args, **kwargs) [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.wait() [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self[:] = self._gt.wait() [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return self._exit_event.wait() [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 809.362826] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] current.throw(*self._exc) [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] result = function(*args, **kwargs) [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return func(*args, **kwargs) [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise e [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] nwinfo = self.network_api.allocate_for_instance( [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] created_port_ids = self._update_ports_for_instance( [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] with excutils.save_and_reraise_exception(): [ 809.363172] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.force_reraise() [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise self.value [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] updated_port = self._update_port( [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] _ensure_no_port_binding_failure(port) [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise exception.PortBindingFailed(port_id=port['id']) [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 809.363463] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] [ 809.363463] env[62736]: INFO nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Terminating instance [ 809.364922] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 809.464420] env[62736]: DEBUG nova.network.neutron [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 809.573821] env[62736]: DEBUG nova.network.neutron [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 809.574364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] Expecting reply to msg 034f71d8cca24c0d984b96089c478977 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 809.583630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 034f71d8cca24c0d984b96089c478977 [ 809.590014] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.634698] env[62736]: DEBUG nova.scheduler.client.report [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 809.637400] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 258d905a19924f2c9cf7168e58661d67 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 809.648899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 258d905a19924f2c9cf7168e58661d67 [ 810.079725] env[62736]: DEBUG oslo_concurrency.lockutils [req-b2e94cab-1249-4fbe-beeb-e5ef9cbfa4a3 req-b22b6937-03b0-4ca9-a3c3-e0666f72b394 service nova] Releasing lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 810.080171] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquired lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 810.080357] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 810.080800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 1b0e969764e649d493044af55cbe0666 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.087487] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b0e969764e649d493044af55cbe0666 [ 810.139690] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.838s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.141988] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.398s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.143773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 0ea01759cff24fc89a515f70759885f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.161132] env[62736]: INFO nova.scheduler.client.report [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Deleted allocations for instance bbc25649-cfa4-4f5f-ab1e-5f1347780612 [ 810.163835] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg 0ebeb730b0564143b6e6037b738849da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.179379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ea01759cff24fc89a515f70759885f4 [ 810.200590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0ebeb730b0564143b6e6037b738849da [ 810.606325] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.670065] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Lock "bbc25649-cfa4-4f5f-ab1e-5f1347780612" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 29.440s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.670412] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5018db37-08be-48fa-b17e-7a1d55d96c95 tempest-ServersAdmin275Test-163722317 tempest-ServersAdmin275Test-163722317-project-member] Expecting reply to msg b647e1684e2b4ca0bb5f6aaf0bf4ff8f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.686355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b647e1684e2b4ca0bb5f6aaf0bf4ff8f [ 810.724280] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 810.724808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg dbb4854189134a669e4c463149b48db4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.741501] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbb4854189134a669e4c463149b48db4 [ 810.885472] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b751a5a-6b08-450d-9d71-6b5e3539629a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.892915] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e768b58d-75f4-4a3c-a72a-386b476a7594 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.922778] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcc367d4-3482-490c-9ec4-4b3b1ec42a23 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.930036] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71deed71-897f-468a-a970-ecd5c282662c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 810.943203] env[62736]: DEBUG nova.compute.provider_tree [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 810.943699] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg fa7dc6e3d6d1436d82585e17ea6b47f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 810.950784] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa7dc6e3d6d1436d82585e17ea6b47f1 [ 810.997791] env[62736]: DEBUG nova.compute.manager [req-413125a2-4ee0-4e35-a4cd-d0768f643916 req-2e6bd5fc-129d-4281-af00-caa3084da711 service nova] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Received event network-vif-deleted-6696136d-efd5-48c3-84b6-c8fcd9d3aa8a {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 811.227047] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Releasing lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 811.227469] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 811.227700] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 811.228096] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-de9176e8-27b3-4c2b-9eba-4aab101c347f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.237085] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e70bbd0-37e6-44d0-9dba-ceda415a01fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 811.259530] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 99735498-3c85-4a01-bc05-4def014618a8 could not be found. [ 811.259765] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 811.259938] env[62736]: INFO nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Took 0.03 seconds to destroy the instance on the hypervisor. [ 811.260213] env[62736]: DEBUG oslo.service.loopingcall [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 811.260436] env[62736]: DEBUG nova.compute.manager [-] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 811.260543] env[62736]: DEBUG nova.network.neutron [-] [instance: 99735498-3c85-4a01-bc05-4def014618a8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 811.276955] env[62736]: DEBUG nova.network.neutron [-] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 811.277481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3e4d0b5d68504a5282ab505394eb6749 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 811.284977] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e4d0b5d68504a5282ab505394eb6749 [ 811.448436] env[62736]: DEBUG nova.scheduler.client.report [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 811.450889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d6a6677fb42342eeabb999835703734b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 811.470222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6a6677fb42342eeabb999835703734b [ 811.779467] env[62736]: DEBUG nova.network.neutron [-] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 811.779956] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 737f32ec217b48daab266bc0bdefa3fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 811.791257] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 737f32ec217b48daab266bc0bdefa3fe [ 811.953473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.811s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 811.954088] env[62736]: ERROR nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Traceback (most recent call last): [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.driver.spawn(context, instance, image_meta, [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] vm_ref = self.build_virtual_machine(instance, [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] vif_infos = vmwarevif.get_vif_info(self._session, [ 811.954088] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] for vif in network_info: [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return self._sync_wrapper(fn, *args, **kwargs) [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.wait() [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self[:] = self._gt.wait() [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return self._exit_event.wait() [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] current.throw(*self._exc) [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 811.954462] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] result = function(*args, **kwargs) [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] return func(*args, **kwargs) [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise e [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] nwinfo = self.network_api.allocate_for_instance( [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] created_port_ids = self._update_ports_for_instance( [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] with excutils.save_and_reraise_exception(): [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] self.force_reraise() [ 811.954815] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise self.value [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] updated_port = self._update_port( [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] _ensure_no_port_binding_failure(port) [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] raise exception.PortBindingFailed(port_id=port['id']) [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] nova.exception.PortBindingFailed: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. [ 811.955168] env[62736]: ERROR nova.compute.manager [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] [ 811.955168] env[62736]: DEBUG nova.compute.utils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 811.955969] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 18.380s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 811.957760] env[62736]: INFO nova.compute.claims [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 811.959301] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg b43c4552f9744ce188df86eb9bdb4e54 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 811.960971] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Build of instance e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0 was re-scheduled: Binding failed for port 1a7e7d7a-929e-4293-87d4-20021d6f1e74, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 811.961436] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 811.961675] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 811.961814] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 811.961978] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 811.962479] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 28188ac289644e10b2bd5e5983db78db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 811.968725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28188ac289644e10b2bd5e5983db78db [ 812.009259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b43c4552f9744ce188df86eb9bdb4e54 [ 812.282225] env[62736]: INFO nova.compute.manager [-] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Took 1.02 seconds to deallocate network for instance. [ 812.284514] env[62736]: DEBUG nova.compute.claims [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 812.284760] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 812.465936] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg fe8634dda60b49ee8aa27b597d411d4f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 812.477649] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe8634dda60b49ee8aa27b597d411d4f [ 812.491555] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.737428] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 812.737932] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 87714ab090d04b78bc13c5d2d40901b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 812.753614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87714ab090d04b78bc13c5d2d40901b9 [ 813.246501] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 813.246756] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 813.246907] env[62736]: DEBUG nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 813.247063] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 813.251435] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c324bac8-b27f-4d29-97e3-16936cd9258e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.259704] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa6cc841-6be8-4744-8c10-e298e21e49f4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.294008] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.294642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg ab4632c47c1a4490a89feef7472d5949 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 813.297494] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a5f63a-6f70-4f3b-bf04-0c5deb634584 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.301800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab4632c47c1a4490a89feef7472d5949 [ 813.305878] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef3f9f5e-9dbd-4409-96eb-861c7265f96b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 813.320205] env[62736]: DEBUG nova.compute.provider_tree [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 813.320799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 11a771b104064e839f2b520d3230b292 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 813.328209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11a771b104064e839f2b520d3230b292 [ 813.699902] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquiring lock "7d7cc825-7dcc-4b21-bf4c-886de451863a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 813.700177] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "7d7cc825-7dcc-4b21-bf4c-886de451863a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 813.801300] env[62736]: DEBUG nova.network.neutron [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.801831] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg f4f37f70e56a45278547182cf59e7775 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 813.810001] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4f37f70e56a45278547182cf59e7775 [ 813.823004] env[62736]: DEBUG nova.scheduler.client.report [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 813.825595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg bb5b1bada20d4f3b9032bf8eaced5d9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 813.841969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb5b1bada20d4f3b9032bf8eaced5d9a [ 814.303657] env[62736]: INFO nova.compute.manager [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0] Took 1.06 seconds to deallocate network for instance. [ 814.305362] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 5f187a0ff41f4bf8a0a7f24c57626938 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 814.328622] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.372s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 814.329137] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 814.330772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 6d70ec5f2add4ee29989568eb5cf8b49 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 814.331750] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.010s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 814.333665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 4decfab3f0574a52ac7e98ae8a3c206c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 814.357859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f187a0ff41f4bf8a0a7f24c57626938 [ 814.369722] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d70ec5f2add4ee29989568eb5cf8b49 [ 814.374296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4decfab3f0574a52ac7e98ae8a3c206c [ 814.810283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 138551a7e48d45abb8ac2575c81ecd05 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 814.836947] env[62736]: DEBUG nova.compute.utils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 814.837870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 51d8ffe4df0d47db850ae20a31c5d6cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 814.842375] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 814.842584] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 814.847797] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 138551a7e48d45abb8ac2575c81ecd05 [ 814.863258] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51d8ffe4df0d47db850ae20a31c5d6cf [ 814.949318] env[62736]: DEBUG nova.policy [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '303068b2037f422b9cba37aebaca71ab', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9703f47d3d944740a48535c270de37ed', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 815.141740] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2c2baa7-a564-4a7f-87ab-8030042ae8e5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.150811] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3a5ab259-d0f6-4de1-8457-d4a13b74cb8f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.182490] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aabe45a5-e6b8-4287-b6fa-2c72c376e423 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.196853] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aff92d60-2a98-455d-ae5d-4186ef39c3e0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 815.212662] env[62736]: DEBUG nova.compute.provider_tree [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 815.212662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg ff9e1212b5984d93890cc4be75106700 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.219098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff9e1212b5984d93890cc4be75106700 [ 815.343174] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 815.345371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg f34801eb526f4ba2abe7a3a8186ce5ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.348288] env[62736]: INFO nova.scheduler.client.report [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Deleted allocations for instance e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0 [ 815.354448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 0d8327ea7ed848bd9ebde27c49992e98 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.395589] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Successfully created port: 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 815.399405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d8327ea7ed848bd9ebde27c49992e98 [ 815.404924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f34801eb526f4ba2abe7a3a8186ce5ca [ 815.494705] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquiring lock "86be55c9-82d5-4a34-b628-4729c42d83c1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.494942] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "86be55c9-82d5-4a34-b628-4729c42d83c1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.720369] env[62736]: DEBUG nova.scheduler.client.report [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 815.722814] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 8d81b253e03943bc9c22cc013a83fa3c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.737144] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d81b253e03943bc9c22cc013a83fa3c [ 815.850717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 90d9548b029441d6bd7f9fdfb1aad832 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.856489] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6652971a-9d51-41c0-9fb9-f9a8cf53e955 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "e7b5d223-a1c7-4b17-a14b-9d75deeb3ac0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 169.421s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.857104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e789619441924b0d9df54395e2ca7d36 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 815.874260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e789619441924b0d9df54395e2ca7d36 [ 815.890215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90d9548b029441d6bd7f9fdfb1aad832 [ 816.225888] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.894s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.226535] env[62736]: ERROR nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Traceback (most recent call last): [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.driver.spawn(context, instance, image_meta, [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] vm_ref = self.build_virtual_machine(instance, [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] vif_infos = vmwarevif.get_vif_info(self._session, [ 816.226535] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] for vif in network_info: [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return self._sync_wrapper(fn, *args, **kwargs) [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.wait() [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self[:] = self._gt.wait() [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return self._exit_event.wait() [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] current.throw(*self._exc) [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 816.226842] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] result = function(*args, **kwargs) [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] return func(*args, **kwargs) [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise e [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] nwinfo = self.network_api.allocate_for_instance( [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] created_port_ids = self._update_ports_for_instance( [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] with excutils.save_and_reraise_exception(): [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] self.force_reraise() [ 816.227255] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise self.value [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] updated_port = self._update_port( [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] _ensure_no_port_binding_failure(port) [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] raise exception.PortBindingFailed(port_id=port['id']) [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] nova.exception.PortBindingFailed: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. [ 816.227576] env[62736]: ERROR nova.compute.manager [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] [ 816.227576] env[62736]: DEBUG nova.compute.utils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 816.228473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 18.330s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.230387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 5e09c1388ba84cdda85894481bb9e9a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 816.231488] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Build of instance fd2547e6-9449-4839-8ebb-f02f01a85b8c was re-scheduled: Binding failed for port e0c0a849-c8ce-4d35-a085-a6276922400b, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 816.231902] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 816.232145] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 816.232292] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 816.232448] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 816.232921] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 648f0963d4064a14bbd9548787fc1fd9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 816.239930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 648f0963d4064a14bbd9548787fc1fd9 [ 816.263125] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e09c1388ba84cdda85894481bb9e9a1 [ 816.353696] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 816.359260] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 816.367027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e1197870dcd746dabeeefa6e0f656e28 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 816.395607] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 816.395851] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 816.396049] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 816.396206] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 816.396349] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 816.396611] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 816.396850] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 816.397023] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 816.397206] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 816.397373] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 816.397547] env[62736]: DEBUG nova.virt.hardware [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 816.398407] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be67b91b-47e8-4d5c-9d01-2aa670c04dc2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.401459] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1197870dcd746dabeeefa6e0f656e28 [ 816.414611] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bac7ec7-7b26-4fa5-b731-a8321ace2cb5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.456258] env[62736]: DEBUG nova.compute.manager [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Received event network-changed-902fe20e-4e9c-49b8-b7b0-c5b27cf5de23 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 816.456441] env[62736]: DEBUG nova.compute.manager [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Refreshing instance network info cache due to event network-changed-902fe20e-4e9c-49b8-b7b0-c5b27cf5de23. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 816.456650] env[62736]: DEBUG oslo_concurrency.lockutils [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] Acquiring lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 816.456783] env[62736]: DEBUG oslo_concurrency.lockutils [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] Acquired lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 816.457014] env[62736]: DEBUG nova.network.neutron [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Refreshing network info cache for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 816.457477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] Expecting reply to msg ae1625416b8a46af867ffa3b6f84c4bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 816.468053] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae1625416b8a46af867ffa3b6f84c4bf [ 816.699452] env[62736]: ERROR nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 816.699452] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 816.699452] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 816.699452] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 816.699452] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 816.699452] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 816.699452] env[62736]: ERROR nova.compute.manager raise self.value [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 816.699452] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 816.699452] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 816.699452] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 816.699891] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 816.699891] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 816.699891] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 816.699891] env[62736]: ERROR nova.compute.manager [ 816.700247] env[62736]: Traceback (most recent call last): [ 816.700340] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 816.700340] env[62736]: listener.cb(fileno) [ 816.700452] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 816.700452] env[62736]: result = function(*args, **kwargs) [ 816.700518] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 816.700518] env[62736]: return func(*args, **kwargs) [ 816.700583] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 816.700583] env[62736]: raise e [ 816.700647] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 816.700647] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 816.700713] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 816.700713] env[62736]: created_port_ids = self._update_ports_for_instance( [ 816.700777] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 816.700777] env[62736]: with excutils.save_and_reraise_exception(): [ 816.700848] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 816.700848] env[62736]: self.force_reraise() [ 816.700907] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 816.700907] env[62736]: raise self.value [ 816.700971] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 816.700971] env[62736]: updated_port = self._update_port( [ 816.701035] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 816.701035] env[62736]: _ensure_no_port_binding_failure(port) [ 816.701098] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 816.701098] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 816.701178] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 816.701225] env[62736]: Removing descriptor: 17 [ 816.702006] env[62736]: ERROR nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Traceback (most recent call last): [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] yield resources [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.driver.spawn(context, instance, image_meta, [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self._vmops.spawn(context, instance, image_meta, injected_files, [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] vm_ref = self.build_virtual_machine(instance, [ 816.702006] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] vif_infos = vmwarevif.get_vif_info(self._session, [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] for vif in network_info: [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self._sync_wrapper(fn, *args, **kwargs) [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.wait() [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self[:] = self._gt.wait() [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self._exit_event.wait() [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 816.702351] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] result = hub.switch() [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self.greenlet.switch() [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] result = function(*args, **kwargs) [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return func(*args, **kwargs) [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise e [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] nwinfo = self.network_api.allocate_for_instance( [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] created_port_ids = self._update_ports_for_instance( [ 816.702707] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] with excutils.save_and_reraise_exception(): [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.force_reraise() [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise self.value [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] updated_port = self._update_port( [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] _ensure_no_port_binding_failure(port) [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise exception.PortBindingFailed(port_id=port['id']) [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 816.703048] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] [ 816.703700] env[62736]: INFO nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Terminating instance [ 816.705747] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 816.760505] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 816.840584] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 816.841107] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 0214156030934d648067b54f8d8a032c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 816.851593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0214156030934d648067b54f8d8a032c [ 816.879715] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.950013] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "53625c0d-203a-45f0-b899-5d2673413a53" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.950246] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "53625c0d-203a-45f0-b899-5d2673413a53" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.974374] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e473ee10-adda-48b3-b69d-2aa8f09764ca {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.982343] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff60a08d-f470-4dac-aefe-80bd6e9726ad {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 816.985776] env[62736]: DEBUG nova.network.neutron [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 817.015957] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c0987b8-fa1c-40a5-a9bc-63f7ad105d27 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.023771] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c293923c-4746-4df8-b7b7-a48064a40414 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 817.038898] env[62736]: DEBUG nova.compute.provider_tree [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 817.039446] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 9540380322a743be86ae932ebb91ac32 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.046714] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9540380322a743be86ae932ebb91ac32 [ 817.072790] env[62736]: DEBUG nova.network.neutron [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 817.073288] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] Expecting reply to msg ed709ad8a45343b3b57c7e0c4b713406 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.083157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed709ad8a45343b3b57c7e0c4b713406 [ 817.343384] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-fd2547e6-9449-4839-8ebb-f02f01a85b8c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 817.343599] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 817.343780] env[62736]: DEBUG nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 817.343943] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 817.362671] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 817.363185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 6f32caf320ab4c508dc24212bc6d4b5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.372997] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f32caf320ab4c508dc24212bc6d4b5c [ 817.542478] env[62736]: DEBUG nova.scheduler.client.report [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 817.544869] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 1d20dea26c17402ba33b616fb2bcea4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.557427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d20dea26c17402ba33b616fb2bcea4d [ 817.576417] env[62736]: DEBUG oslo_concurrency.lockutils [req-d05a08e5-9c65-4187-b2c4-2dc9908daf14 req-de2e7a7f-c913-40c5-b99b-1c061e6876ee service nova] Releasing lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 817.576927] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquired lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 817.577228] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 817.577803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg e8d7770d934b464d866440fd15361733 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.584520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8d7770d934b464d866440fd15361733 [ 817.865073] env[62736]: DEBUG nova.network.neutron [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 817.865894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 54bfa1c559a34dbca308083a1969a2d8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 817.876545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54bfa1c559a34dbca308083a1969a2d8 [ 818.047471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.819s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 818.048213] env[62736]: ERROR nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Traceback (most recent call last): [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.driver.spawn(context, instance, image_meta, [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] vm_ref = self.build_virtual_machine(instance, [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] vif_infos = vmwarevif.get_vif_info(self._session, [ 818.048213] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] for vif in network_info: [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return self._sync_wrapper(fn, *args, **kwargs) [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.wait() [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self[:] = self._gt.wait() [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return self._exit_event.wait() [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] current.throw(*self._exc) [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 818.048561] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] result = function(*args, **kwargs) [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] return func(*args, **kwargs) [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise e [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] nwinfo = self.network_api.allocate_for_instance( [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] created_port_ids = self._update_ports_for_instance( [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] with excutils.save_and_reraise_exception(): [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] self.force_reraise() [ 818.048923] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise self.value [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] updated_port = self._update_port( [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] _ensure_no_port_binding_failure(port) [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] raise exception.PortBindingFailed(port_id=port['id']) [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] nova.exception.PortBindingFailed: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. [ 818.049331] env[62736]: ERROR nova.compute.manager [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] [ 818.049331] env[62736]: DEBUG nova.compute.utils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 818.050128] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 17.782s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 818.051937] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 981d791f17a54f90a6d5c3464d154085 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.062862] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Build of instance 5014a870-32fd-4166-9446-c41ed2a4954e was re-scheduled: Binding failed for port 30f94bf9-c081-4c9e-8c94-4a50ed6b7af0, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 818.063327] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 818.063558] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 818.063708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquired lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 818.063864] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 818.064269] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 6d0d72f4287147e1ac83b9fd266054a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.070543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d0d72f4287147e1ac83b9fd266054a7 [ 818.094295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 981d791f17a54f90a6d5c3464d154085 [ 818.101201] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.204868] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 818.205411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg f21b3db549944472a10b2529636622dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.213845] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f21b3db549944472a10b2529636622dd [ 818.368879] env[62736]: INFO nova.compute.manager [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: fd2547e6-9449-4839-8ebb-f02f01a85b8c] Took 1.02 seconds to deallocate network for instance. [ 818.370838] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 00b7a24834944989b353b49362128966 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.403576] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00b7a24834944989b353b49362128966 [ 818.480075] env[62736]: DEBUG nova.compute.manager [req-b35bd50d-071e-40d3-b734-6bba0026702c req-3598447b-bb28-4c26-a721-0d54b932a751 service nova] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Received event network-vif-deleted-902fe20e-4e9c-49b8-b7b0-c5b27cf5de23 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 818.583039] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.673331] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 818.673837] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 339984325a0e450e970bd5724944e3df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.687190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 339984325a0e450e970bd5724944e3df [ 818.707247] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Releasing lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 818.707666] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 818.707854] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 818.708185] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2932628e-4fb6-4a19-9899-8c102baf2632 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.719706] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aee0834-35c4-4039-941c-7c22f62d8025 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.760284] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6c4ac72f-1636-4f4c-928c-0a6fe895ce37 could not be found. [ 818.760565] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 818.760831] env[62736]: INFO nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Took 0.05 seconds to destroy the instance on the hypervisor. [ 818.761134] env[62736]: DEBUG oslo.service.loopingcall [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 818.761632] env[62736]: DEBUG nova.compute.manager [-] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 818.761731] env[62736]: DEBUG nova.network.neutron [-] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 818.774035] env[62736]: DEBUG nova.network.neutron [-] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.774477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b3818adae4c349c2b0e5bb3ec97d92e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.782875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3818adae4c349c2b0e5bb3ec97d92e1 [ 818.837168] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf4d0cba-b60b-482d-8bb0-0f8cff5161a6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.844651] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65bfd262-5338-45fc-a004-64b0d09941cb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.875449] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 5d1f4fd09e7241bd92717f39cdb4764b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.877092] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8876f5c-b21f-4601-99f9-cb3c15887cbf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.884457] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4ed1792-1216-4879-b717-d1ba39d49f44 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 818.897708] env[62736]: DEBUG nova.compute.provider_tree [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 818.898188] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg b133bace329e4a998c670630a0002796 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 818.904317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d1f4fd09e7241bd92717f39cdb4764b [ 818.905427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b133bace329e4a998c670630a0002796 [ 819.175597] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Releasing lock "refresh_cache-5014a870-32fd-4166-9446-c41ed2a4954e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 819.175806] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 819.175980] env[62736]: DEBUG nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 819.176173] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 819.190680] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 819.191287] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg dc8786ef87284bb0aa660052859d79c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.197900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc8786ef87284bb0aa660052859d79c5 [ 819.276967] env[62736]: DEBUG nova.network.neutron [-] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.277140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1aad596190a24c94bb6fcce8fab84f10 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.285143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aad596190a24c94bb6fcce8fab84f10 [ 819.398831] env[62736]: INFO nova.scheduler.client.report [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Deleted allocations for instance fd2547e6-9449-4839-8ebb-f02f01a85b8c [ 819.404828] env[62736]: DEBUG nova.scheduler.client.report [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 819.406997] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 815ea193fbcf4c829204a20c023220e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.408040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg a454f3776233409fa158af9854932969 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.417538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 815ea193fbcf4c829204a20c023220e3 [ 819.465391] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a454f3776233409fa158af9854932969 [ 819.693567] env[62736]: DEBUG nova.network.neutron [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.694110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg d0896e659e6b49ffa7df988a54f58f44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.702776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0896e659e6b49ffa7df988a54f58f44 [ 819.780699] env[62736]: INFO nova.compute.manager [-] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Took 1.02 seconds to deallocate network for instance. [ 819.783060] env[62736]: DEBUG nova.compute.claims [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 819.783224] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 819.909911] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.860s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.910586] env[62736]: ERROR nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Traceback (most recent call last): [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.driver.spawn(context, instance, image_meta, [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] vm_ref = self.build_virtual_machine(instance, [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 819.910586] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] for vif in network_info: [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return self._sync_wrapper(fn, *args, **kwargs) [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.wait() [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self[:] = self._gt.wait() [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return self._exit_event.wait() [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] current.throw(*self._exc) [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 819.910965] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] result = function(*args, **kwargs) [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] return func(*args, **kwargs) [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise e [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] nwinfo = self.network_api.allocate_for_instance( [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] created_port_ids = self._update_ports_for_instance( [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] with excutils.save_and_reraise_exception(): [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] self.force_reraise() [ 819.911261] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise self.value [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] updated_port = self._update_port( [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] _ensure_no_port_binding_failure(port) [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] raise exception.PortBindingFailed(port_id=port['id']) [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] nova.exception.PortBindingFailed: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. [ 819.911548] env[62736]: ERROR nova.compute.manager [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] [ 819.911548] env[62736]: DEBUG nova.compute.utils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 819.912955] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Build of instance b49cde53-0ef7-44a4-b894-e37d1cbf41a7 was re-scheduled: Binding failed for port cc0c8455-7089-49be-8598-cb883d892f6d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 819.913346] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 819.913569] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquiring lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 819.913709] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Acquired lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 819.913855] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 819.914247] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg b39133734c3a4a43ab90e498291cbc31 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.915419] env[62736]: DEBUG oslo_concurrency.lockutils [None req-977306e0-8d53-44bd-9f97-fbb2d1b9be52 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "fd2547e6-9449-4839-8ebb-f02f01a85b8c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 157.900s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.915777] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.346s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 819.917180] env[62736]: INFO nova.compute.claims [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 819.918643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg b90589aef0104be3b34d9575fc370d3d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.920544] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 55edf3e2a4af4a78b46085bdeccd4a6e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 819.921899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b39133734c3a4a43ab90e498291cbc31 [ 819.927884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55edf3e2a4af4a78b46085bdeccd4a6e [ 819.968634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b90589aef0104be3b34d9575fc370d3d [ 820.196669] env[62736]: INFO nova.compute.manager [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: 5014a870-32fd-4166-9446-c41ed2a4954e] Took 1.02 seconds to deallocate network for instance. [ 820.198583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 5471cded28bc43c1a9674d962a808f6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 820.232823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5471cded28bc43c1a9674d962a808f6a [ 820.421688] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg e71a09ab067144578d06e589044bef6f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 820.423685] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 820.425418] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg a8f165077e0040188c0f5377ca576160 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 820.429821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e71a09ab067144578d06e589044bef6f [ 820.431293] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 820.457664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8f165077e0040188c0f5377ca576160 [ 820.514128] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 820.514671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg b1e0ca835bdf401f928f6b4668dfd2f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 820.522991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1e0ca835bdf401f928f6b4668dfd2f1 [ 820.703985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg a5168b523ba846f588ef269b269adfe9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 820.737705] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5168b523ba846f588ef269b269adfe9 [ 820.942139] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.018222] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Releasing lock "refresh_cache-b49cde53-0ef7-44a4-b894-e37d1cbf41a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 821.018452] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 821.018633] env[62736]: DEBUG nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 821.018801] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 821.042207] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 821.043215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg acc26cad8dd84c6982dfb73b89b85349 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.050253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acc26cad8dd84c6982dfb73b89b85349 [ 821.188527] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c581e0ef-6f47-4cba-a88f-7e6e87d4d7ae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.197075] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cdeb8c29-4943-4ac3-a514-db1cde68d27d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.238203] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f5b3330d-739d-4b88-8e9f-11f992d858f8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.245986] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7cb79e8-db08-4363-96d9-51b2ad7c529d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 821.250892] env[62736]: INFO nova.scheduler.client.report [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Deleted allocations for instance 5014a870-32fd-4166-9446-c41ed2a4954e [ 821.257061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 1786b60e43be4a9e9540f958fb4dafd2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.266050] env[62736]: DEBUG nova.compute.provider_tree [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 821.266540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 14c1232ed2654d95ae22944a6db69caf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.272599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1786b60e43be4a9e9540f958fb4dafd2 [ 821.280987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14c1232ed2654d95ae22944a6db69caf [ 821.548699] env[62736]: DEBUG nova.network.neutron [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 821.549190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 7a4ce67e6d8b4cb0a20a6572ba877fdf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.558619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a4ce67e6d8b4cb0a20a6572ba877fdf [ 821.758964] env[62736]: DEBUG oslo_concurrency.lockutils [None req-383ce570-73cb-447d-a3b6-a6192d1a0d22 tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "5014a870-32fd-4166-9446-c41ed2a4954e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 150.106s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.759633] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6534888ed27e42a49574791a1ad29106 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.769063] env[62736]: DEBUG nova.scheduler.client.report [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 821.771655] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 1c274cdc492e4a28bfc7f67a3a177d6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 821.773523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6534888ed27e42a49574791a1ad29106 [ 821.787744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c274cdc492e4a28bfc7f67a3a177d6a [ 822.051547] env[62736]: INFO nova.compute.manager [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] [instance: b49cde53-0ef7-44a4-b894-e37d1cbf41a7] Took 1.03 seconds to deallocate network for instance. [ 822.053716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg bf6fc52dad3d42e8b0312a8da9a3bd26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.090624] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf6fc52dad3d42e8b0312a8da9a3bd26 [ 822.262402] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 822.264293] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6b79cf7a08ed448dade7753718766451 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.277215] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.361s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 822.277547] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 822.279159] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg a129765999014682aabf7c54e68ba3cc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.280573] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.798s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 822.285476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 08c91a69bab944a386a28bb55fc8a25e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.314940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a129765999014682aabf7c54e68ba3cc [ 822.336245] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b79cf7a08ed448dade7753718766451 [ 822.344959] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08c91a69bab944a386a28bb55fc8a25e [ 822.558926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 3f9aba11e6b24b66bb60930757c038b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.591097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f9aba11e6b24b66bb60930757c038b0 [ 822.781454] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 822.786017] env[62736]: DEBUG nova.compute.utils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 822.786017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 8f3bce9b3d6849d2aaa8c9e48c9dc09a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 822.789610] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 822.789774] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 822.796375] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f3bce9b3d6849d2aaa8c9e48c9dc09a [ 822.836679] env[62736]: DEBUG nova.policy [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '16306c5f0e4d4eabbcce4a9c72edce5f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '532d5738736643d2836c0896012494da', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 822.967094] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 822.967094] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 823.030315] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ac701c4-d1aa-4ac9-82c7-6d1c14bc2dd8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.037974] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d8c4c56-6dbb-4caf-a5dd-cd6026da9d0d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.070020] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ccb0028-2494-4854-81ed-fd60dfb2daa6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.078003] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc4179d3-739d-4b05-ab7a-ee2cf1c575f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 823.084428] env[62736]: INFO nova.scheduler.client.report [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Deleted allocations for instance b49cde53-0ef7-44a4-b894-e37d1cbf41a7 [ 823.092170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Expecting reply to msg 88093ce8a50f4dbc82928ca0624fb3bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.103100] env[62736]: DEBUG nova.compute.provider_tree [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 823.103100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 62d66d9f768a419595e5f9a31c5cfd0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.103716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88093ce8a50f4dbc82928ca0624fb3bb [ 823.117191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62d66d9f768a419595e5f9a31c5cfd0a [ 823.184259] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Successfully created port: 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 823.290799] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 823.292530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 41b728ab38374dccb045795e04504d3f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.315702] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "570ee4e0-caeb-41f9-82d2-fa009f23581d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 823.315919] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "570ee4e0-caeb-41f9-82d2-fa009f23581d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 823.323726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41b728ab38374dccb045795e04504d3f [ 823.594593] env[62736]: DEBUG oslo_concurrency.lockutils [None req-eabfdd60-e769-4bf8-86c2-89886d2aa6f5 tempest-InstanceActionsNegativeTestJSON-21200735 tempest-InstanceActionsNegativeTestJSON-21200735-project-member] Lock "b49cde53-0ef7-44a4-b894-e37d1cbf41a7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.422s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 823.595359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a913e7692dfb477995dd6411195d27be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.605355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a913e7692dfb477995dd6411195d27be [ 823.606329] env[62736]: DEBUG nova.scheduler.client.report [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 823.608694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg e35681533ebc423eb32009fceb15087b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.620898] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e35681533ebc423eb32009fceb15087b [ 823.797030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 2bbe10d84b0a45c0b257778605c088e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 823.832205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bbe10d84b0a45c0b257778605c088e3 [ 824.098028] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 824.099950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg c5c43937a9154519a33e6b9268db3934 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 824.110823] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.831s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.111529] env[62736]: ERROR nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Traceback (most recent call last): [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.driver.spawn(context, instance, image_meta, [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] vm_ref = self.build_virtual_machine(instance, [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] vif_infos = vmwarevif.get_vif_info(self._session, [ 824.111529] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] for vif in network_info: [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return self._sync_wrapper(fn, *args, **kwargs) [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.wait() [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self[:] = self._gt.wait() [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return self._exit_event.wait() [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] current.throw(*self._exc) [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 824.111846] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] result = function(*args, **kwargs) [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] return func(*args, **kwargs) [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise e [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] nwinfo = self.network_api.allocate_for_instance( [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] created_port_ids = self._update_ports_for_instance( [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] with excutils.save_and_reraise_exception(): [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] self.force_reraise() [ 824.112160] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise self.value [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] updated_port = self._update_port( [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] _ensure_no_port_binding_failure(port) [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] raise exception.PortBindingFailed(port_id=port['id']) [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] nova.exception.PortBindingFailed: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. [ 824.112457] env[62736]: ERROR nova.compute.manager [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] [ 824.112457] env[62736]: DEBUG nova.compute.utils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 824.113585] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.335s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.115235] env[62736]: INFO nova.compute.claims [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 824.116766] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 93324a966e3d4e6da4b2855eb1d87153 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 824.117926] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Build of instance 25e906f6-ec98-4c99-83c7-43f94c5790dd was re-scheduled: Binding failed for port 77e7b973-54f1-4090-9007-9bdb448043dd, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 824.118349] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 824.118583] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 824.118771] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 824.118862] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 824.119199] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 4a1f8ef9e2294a18820d86e1e10de7d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 824.132679] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a1f8ef9e2294a18820d86e1e10de7d6 [ 824.932367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5c43937a9154519a33e6b9268db3934 [ 824.932898] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93324a966e3d4e6da4b2855eb1d87153 [ 824.934741] env[62736]: ERROR nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 824.934741] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 824.934741] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 824.934741] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 824.934741] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 824.934741] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 824.934741] env[62736]: ERROR nova.compute.manager raise self.value [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 824.934741] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 824.934741] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 824.934741] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 824.935147] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 824.935147] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 824.935147] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 824.935147] env[62736]: ERROR nova.compute.manager [ 824.935147] env[62736]: Traceback (most recent call last): [ 824.935147] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 824.935147] env[62736]: listener.cb(fileno) [ 824.935147] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 824.935147] env[62736]: result = function(*args, **kwargs) [ 824.935147] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 824.935147] env[62736]: return func(*args, **kwargs) [ 824.935147] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 824.935147] env[62736]: raise e [ 824.935147] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 824.935147] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 824.935147] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 824.935147] env[62736]: created_port_ids = self._update_ports_for_instance( [ 824.935147] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 824.935147] env[62736]: with excutils.save_and_reraise_exception(): [ 824.935147] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 824.935147] env[62736]: self.force_reraise() [ 824.935147] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 824.935147] env[62736]: raise self.value [ 824.935147] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 824.935147] env[62736]: updated_port = self._update_port( [ 824.935147] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 824.935147] env[62736]: _ensure_no_port_binding_failure(port) [ 824.935147] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 824.935147] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 824.935791] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 824.935791] env[62736]: Removing descriptor: 17 [ 824.935912] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 824.940605] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg d98da18e7b594654b6d94781bc41cea0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 824.945145] env[62736]: DEBUG nova.compute.manager [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Received event network-changed-6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 824.945145] env[62736]: DEBUG nova.compute.manager [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Refreshing instance network info cache due to event network-changed-6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 824.945145] env[62736]: DEBUG oslo_concurrency.lockutils [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] Acquiring lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 824.945145] env[62736]: DEBUG oslo_concurrency.lockutils [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] Acquired lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 824.945145] env[62736]: DEBUG nova.network.neutron [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Refreshing network info cache for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 824.945381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] Expecting reply to msg 1aaa6517291146a1bf2e9a9c33b4b72a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 824.948779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d98da18e7b594654b6d94781bc41cea0 [ 824.953849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aaa6517291146a1bf2e9a9c33b4b72a [ 824.968910] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 824.984526] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 824.984526] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 824.984526] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 824.985232] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 824.985232] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 824.985232] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 824.985232] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 824.985232] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 824.985599] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 824.985599] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 824.985599] env[62736]: DEBUG nova.virt.hardware [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 824.985599] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-798eed32-af3c-4a1b-88ea-29bc99687286 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.985599] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.985872] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3116a4f-6422-4c2d-ba91-3fa7df07cc1d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 824.996637] env[62736]: ERROR nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Traceback (most recent call last): [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] yield resources [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.driver.spawn(context, instance, image_meta, [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] vm_ref = self.build_virtual_machine(instance, [ 824.996637] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] vif_infos = vmwarevif.get_vif_info(self._session, [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] for vif in network_info: [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return self._sync_wrapper(fn, *args, **kwargs) [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.wait() [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self[:] = self._gt.wait() [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return self._exit_event.wait() [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 824.997202] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] current.throw(*self._exc) [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] result = function(*args, **kwargs) [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return func(*args, **kwargs) [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise e [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] nwinfo = self.network_api.allocate_for_instance( [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] created_port_ids = self._update_ports_for_instance( [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] with excutils.save_and_reraise_exception(): [ 824.997529] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.force_reraise() [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise self.value [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] updated_port = self._update_port( [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] _ensure_no_port_binding_failure(port) [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise exception.PortBindingFailed(port_id=port['id']) [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 824.997852] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] [ 824.997852] env[62736]: INFO nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Terminating instance [ 824.998909] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 825.081589] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 825.082104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 98b15b373a924d8ca1545af026f364dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 825.089758] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98b15b373a924d8ca1545af026f364dd [ 825.470705] env[62736]: DEBUG nova.network.neutron [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.564022] env[62736]: DEBUG nova.network.neutron [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 825.564543] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] Expecting reply to msg 82aa16e468fe4bcc94f5c1fcedf94889 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 825.574489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82aa16e468fe4bcc94f5c1fcedf94889 [ 825.575599] env[62736]: DEBUG oslo_concurrency.lockutils [req-4230dd6c-19af-440c-8be3-1ed682399a09 req-954f4ff5-5f49-4d5e-b5d6-b361d49b78f0 service nova] Releasing lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 825.575937] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquired lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 825.576139] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 825.576521] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg fd2b0def378544c697ed465774b94414 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 825.584165] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-25e906f6-ec98-4c99-83c7-43f94c5790dd" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 825.584243] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 825.585316] env[62736]: DEBUG nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 825.585316] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 825.588055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd2b0def378544c697ed465774b94414 [ 825.623506] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.624162] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 347647722c284cfca1cb0de40f50393e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 825.634243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 347647722c284cfca1cb0de40f50393e [ 825.758935] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d80486c7-5949-491b-a9c4-8e9060486733 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.772422] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea9547ff-8fb5-404e-bff0-8c458a5b3c53 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.807780] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33edc025-cbed-468c-8b51-ae991c36a391 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.815974] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf1d26bc-1bc1-4ae8-b580-55e43c28ace9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 825.830581] env[62736]: DEBUG nova.compute.provider_tree [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 825.831157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 76096ec9dc7a41418248e4dd2f7bd7d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 825.839259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76096ec9dc7a41418248e4dd2f7bd7d1 [ 826.102714] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 826.131394] env[62736]: DEBUG nova.network.neutron [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.131964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 171a461234664997914515f6f8fe53bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.139854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 171a461234664997914515f6f8fe53bd [ 826.180955] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.181528] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg a767814670464caeb120b07a30d98f51 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.189928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a767814670464caeb120b07a30d98f51 [ 826.270242] env[62736]: DEBUG nova.compute.manager [req-7461139e-3abe-4a51-8337-d8000defa886 req-91d180c6-34bf-4df7-9fc4-68faf71f47ae service nova] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Received event network-vif-deleted-6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 826.337702] env[62736]: DEBUG nova.scheduler.client.report [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 826.340195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 6e5e1f114a7d44028250960ac5e592a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.355386] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e5e1f114a7d44028250960ac5e592a1 [ 826.634471] env[62736]: INFO nova.compute.manager [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: 25e906f6-ec98-4c99-83c7-43f94c5790dd] Took 1.05 seconds to deallocate network for instance. [ 826.636318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 87e9b1a75074442f906de700756bc173 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.672410] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87e9b1a75074442f906de700756bc173 [ 826.683319] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Releasing lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 826.683762] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 826.683956] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 826.684268] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-076fdf4d-315c-4862-aed7-5c503c930d86 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.694258] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54b9e064-2037-4f78-8fbd-183dbc12201d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 826.715262] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 451b9e70-f202-4a10-8b19-34b6167ef14d could not be found. [ 826.715509] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 826.715688] env[62736]: INFO nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 826.715923] env[62736]: DEBUG oslo.service.loopingcall [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 826.716195] env[62736]: DEBUG nova.compute.manager [-] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 826.716288] env[62736]: DEBUG nova.network.neutron [-] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 826.736746] env[62736]: DEBUG nova.network.neutron [-] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 826.737263] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg df5a729ba66a40dcb8e36207ca847443 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.743986] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df5a729ba66a40dcb8e36207ca847443 [ 826.843188] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.729s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.843780] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 826.846994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg b9232a134f8f492ab29045fc9daccc91 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.848039] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 17.258s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.849385] env[62736]: INFO nova.compute.claims [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 826.850871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg cde454ffcf9045ac84a2aee7bd3c9772 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 826.889208] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cde454ffcf9045ac84a2aee7bd3c9772 [ 826.896867] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9232a134f8f492ab29045fc9daccc91 [ 827.141223] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 5289c6996d4f4730a847e94debfadb21 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.171528] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5289c6996d4f4730a847e94debfadb21 [ 827.239625] env[62736]: DEBUG nova.network.neutron [-] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.240096] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a536fc2145cc4009bb782428d9987ee6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.248673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a536fc2145cc4009bb782428d9987ee6 [ 827.353979] env[62736]: DEBUG nova.compute.utils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 827.354779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 9e7acf01210440a0a133787a4cf4ae22 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.356881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg d9593595e7ca40279c50e8ab3b825670 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.358238] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 827.367597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9593595e7ca40279c50e8ab3b825670 [ 827.368630] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e7acf01210440a0a133787a4cf4ae22 [ 827.665286] env[62736]: INFO nova.scheduler.client.report [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Deleted allocations for instance 25e906f6-ec98-4c99-83c7-43f94c5790dd [ 827.671279] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg c4e952f3cc104ad6a49d09d752db98a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.688027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4e952f3cc104ad6a49d09d752db98a7 [ 827.743454] env[62736]: INFO nova.compute.manager [-] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Took 1.03 seconds to deallocate network for instance. [ 827.744869] env[62736]: DEBUG nova.compute.claims [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 827.744869] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.859923] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 827.861359] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 2cac793ac9c246669b630dcd6af544d4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 827.901965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2cac793ac9c246669b630dcd6af544d4 [ 828.114596] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2abf0127-1b7f-4b15-83e9-31eab0ecddbe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.125619] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09d4fe78-5812-4487-a51e-7ed5c65e5e57 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.157043] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e03fd451-f7e8-49c9-8852-9891aa638779 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.164612] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10db113-ae41-438f-b626-b5f45db39a8e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.177824] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f3bb6349-5815-4d43-914d-567184e1cac0 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "25e906f6-ec98-4c99-83c7-43f94c5790dd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 137.573s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.179166] env[62736]: DEBUG nova.compute.provider_tree [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 828.179166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg c42ecd05b0e747b6816310d8bfba15cc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 828.179846] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg b5e5baf34a3c4d91a783f1854daf3fd6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 828.186197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c42ecd05b0e747b6816310d8bfba15cc [ 828.194170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5e5baf34a3c4d91a783f1854daf3fd6 [ 828.380332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 8931427849b24ca88ffe8f7e2c8f0f90 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 828.417726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8931427849b24ca88ffe8f7e2c8f0f90 [ 828.682170] env[62736]: DEBUG nova.scheduler.client.report [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 828.684232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg e14e21f2e41244e3a7acb0d14b72ac80 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 828.685535] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 828.688168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 62861fa6a14348f5aff624402a2e78e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 828.701308] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e14e21f2e41244e3a7acb0d14b72ac80 [ 828.725642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62861fa6a14348f5aff624402a2e78e0 [ 828.883572] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 828.980783] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 828.981029] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 828.981191] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 828.981376] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 828.981519] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 828.981659] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 828.981927] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 828.982315] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 828.982519] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 828.982701] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 828.982874] env[62736]: DEBUG nova.virt.hardware [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 828.983720] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0797777-9127-4a27-a2ef-e8722e3300d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 828.992156] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-406e427b-5d32-4c9d-9db9-ca737de54bdf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.005638] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 829.011122] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Creating folder: Project (77e1b9d5d344410899f56a473155e5b2). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 829.011635] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-366fe938-e5d1-49f1-b180-32d1d852bf52 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.022270] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Created folder: Project (77e1b9d5d344410899f56a473155e5b2) in parent group-v107913. [ 829.022449] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Creating folder: Instances. Parent ref: group-v107931. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 829.022671] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d1f2d22d-2933-4a07-9b9d-4c1e438a77d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.030873] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Created folder: Instances in parent group-v107931. [ 829.031102] env[62736]: DEBUG oslo.service.loopingcall [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 829.031282] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 829.031501] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a633d3cd-54f9-4eca-b316-c300a26b7a5c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.048844] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 829.048844] env[62736]: value = "task-397745" [ 829.048844] env[62736]: _type = "Task" [ 829.048844] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 829.056309] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397745, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 829.192060] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.342s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.192060] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 829.193108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg edecd55b2f0c43429188676014fa3d8d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 829.196451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.912s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.198157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg fe13c2a6be4841db9f9cb95699331577 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 829.213940] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.236801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg edecd55b2f0c43429188676014fa3d8d [ 829.237202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe13c2a6be4841db9f9cb95699331577 [ 829.276110] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquiring lock "7d047648-facb-4d58-acf7-a99a3f7adf65" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.276110] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "7d047648-facb-4d58-acf7-a99a3f7adf65" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.559762] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397745, 'name': CreateVM_Task, 'duration_secs': 0.239127} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 829.559927] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 829.561158] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 829.561158] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 829.561158] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 829.561158] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-cf0bcedb-bde4-4f7f-9e22-a9b6ddf1ed9b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 829.565053] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 829.565053] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52114525-fd45-e9d8-318b-664881d4c20a" [ 829.565053] env[62736]: _type = "Task" [ 829.565053] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 829.572328] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52114525-fd45-e9d8-318b-664881d4c20a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 829.697941] env[62736]: DEBUG nova.compute.utils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 829.698590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 16a9fa98d1d1496b90202d476d1b7ce9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 829.699580] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 829.699754] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 829.711128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16a9fa98d1d1496b90202d476d1b7ce9 [ 829.777469] env[62736]: DEBUG nova.policy [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9a71b1891f254423bd5e6bd2be1641ee', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9df640c9ce4d4a7e8a953841d0912936', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 829.998695] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-986ae615-5391-4d0c-8198-907cdb0cc500 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.006435] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3d9d837-5e38-4808-a8a2-de0fe36808c4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.044126] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e5508a1-d735-4e6c-8770-3f3d51bcc3c8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.051385] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9be2a20c-dcad-4d4f-acdf-091c0202466d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.064699] env[62736]: DEBUG nova.compute.provider_tree [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 830.065241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg e653d16d71424fac95fc6a6644049067 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 830.073951] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52114525-fd45-e9d8-318b-664881d4c20a, 'name': SearchDatastore_Task, 'duration_secs': 0.009511} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 830.074445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e653d16d71424fac95fc6a6644049067 [ 830.074877] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 830.075096] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 830.075323] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 830.075464] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 830.075635] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 830.076104] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f8bd8192-0636-4799-a787-d1a8fdba33af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.088081] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 830.088081] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 830.088081] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c5a79453-8287-4cb9-b69f-c1f6b0768d52 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.089798] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 830.089798] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5264e2af-ac18-6246-fab2-2759e402131c" [ 830.089798] env[62736]: _type = "Task" [ 830.089798] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 830.099301] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5264e2af-ac18-6246-fab2-2759e402131c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 830.193546] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Successfully created port: 33c97501-12cc-4d90-b4eb-9d442da54929 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 830.203239] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 830.204537] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 588fb44ef7f0459da4293700d57eb1b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 830.244682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 588fb44ef7f0459da4293700d57eb1b7 [ 830.448220] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "fa27cfe2-6f7f-4397-bcc5-147d6865f404" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.448455] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "fa27cfe2-6f7f-4397-bcc5-147d6865f404" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.570818] env[62736]: DEBUG nova.scheduler.client.report [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 830.573310] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 4632ccea575a4c69ba87714fdab99c4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 830.584365] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4632ccea575a4c69ba87714fdab99c4c [ 830.600037] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5264e2af-ac18-6246-fab2-2759e402131c, 'name': SearchDatastore_Task, 'duration_secs': 0.009319} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 830.600842] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-72f3a5fc-f0b7-4154-af49-b302214f58c5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 830.606023] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 830.606023] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5227fe22-b283-b71d-cedc-694003fa2e21" [ 830.606023] env[62736]: _type = "Task" [ 830.606023] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 830.615463] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5227fe22-b283-b71d-cedc-694003fa2e21, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 830.709276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg d14edae5e89d421e816f3c14729e1137 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 830.748255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d14edae5e89d421e816f3c14729e1137 [ 831.076090] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.880s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.076738] env[62736]: ERROR nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] Traceback (most recent call last): [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.driver.spawn(context, instance, image_meta, [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] vm_ref = self.build_virtual_machine(instance, [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] vif_infos = vmwarevif.get_vif_info(self._session, [ 831.076738] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] for vif in network_info: [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return self._sync_wrapper(fn, *args, **kwargs) [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.wait() [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self[:] = self._gt.wait() [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return self._exit_event.wait() [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] current.throw(*self._exc) [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 831.077072] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] result = function(*args, **kwargs) [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] return func(*args, **kwargs) [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise e [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] nwinfo = self.network_api.allocate_for_instance( [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] created_port_ids = self._update_ports_for_instance( [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] with excutils.save_and_reraise_exception(): [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] self.force_reraise() [ 831.077456] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise self.value [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] updated_port = self._update_port( [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] _ensure_no_port_binding_failure(port) [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] raise exception.PortBindingFailed(port_id=port['id']) [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] nova.exception.PortBindingFailed: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. [ 831.077798] env[62736]: ERROR nova.compute.manager [instance: 99735498-3c85-4a01-bc05-4def014618a8] [ 831.077798] env[62736]: DEBUG nova.compute.utils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 831.078714] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.199s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.080147] env[62736]: INFO nova.compute.claims [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 831.081675] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 14c9ad3655de415ea3686d0061a4c2c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.082930] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Build of instance 99735498-3c85-4a01-bc05-4def014618a8 was re-scheduled: Binding failed for port 6696136d-efd5-48c3-84b6-c8fcd9d3aa8a, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 831.083373] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 831.083594] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 831.083731] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquired lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 831.083880] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 831.084387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg cfc4a712271b46fe883cf0dbdad0ed44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.092423] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfc4a712271b46fe883cf0dbdad0ed44 [ 831.117260] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5227fe22-b283-b71d-cedc-694003fa2e21, 'name': SearchDatastore_Task, 'duration_secs': 0.008267} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 831.117653] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 831.117907] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 831.118183] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e96bfa9b-e3ba-4bb6-b31c-2a8920b02876 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.124957] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 831.124957] env[62736]: value = "task-397746" [ 831.124957] env[62736]: _type = "Task" [ 831.124957] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 831.137567] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397746, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 831.138749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14c9ad3655de415ea3686d0061a4c2c5 [ 831.209131] env[62736]: DEBUG nova.compute.manager [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Received event network-changed-33c97501-12cc-4d90-b4eb-9d442da54929 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 831.209369] env[62736]: DEBUG nova.compute.manager [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Refreshing instance network info cache due to event network-changed-33c97501-12cc-4d90-b4eb-9d442da54929. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 831.210210] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] Acquiring lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 831.210339] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] Acquired lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 831.210542] env[62736]: DEBUG nova.network.neutron [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Refreshing network info cache for port 33c97501-12cc-4d90-b4eb-9d442da54929 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 831.210975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] Expecting reply to msg 0b863a88c866443aa804cc48e2e061f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.212732] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 831.232726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b863a88c866443aa804cc48e2e061f3 [ 831.239239] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 831.239529] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 831.239715] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 831.239916] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 831.240109] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 831.240289] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 831.240509] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 831.240662] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 831.240857] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 831.241058] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 831.241266] env[62736]: DEBUG nova.virt.hardware [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 831.242440] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2603fb13-7e27-4d77-9e86-4d148bc608bd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.251307] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2179f5e-c33d-4aaf-98b0-f5173a23591f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.403897] env[62736]: ERROR nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 831.403897] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 831.403897] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 831.403897] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 831.403897] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 831.403897] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 831.403897] env[62736]: ERROR nova.compute.manager raise self.value [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 831.403897] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 831.403897] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 831.403897] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 831.404692] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 831.404692] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 831.404692] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 831.404692] env[62736]: ERROR nova.compute.manager [ 831.404692] env[62736]: Traceback (most recent call last): [ 831.404692] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 831.404692] env[62736]: listener.cb(fileno) [ 831.404692] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 831.404692] env[62736]: result = function(*args, **kwargs) [ 831.404692] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 831.404692] env[62736]: return func(*args, **kwargs) [ 831.404692] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 831.404692] env[62736]: raise e [ 831.404692] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 831.404692] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 831.404692] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 831.404692] env[62736]: created_port_ids = self._update_ports_for_instance( [ 831.404692] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 831.404692] env[62736]: with excutils.save_and_reraise_exception(): [ 831.404692] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 831.404692] env[62736]: self.force_reraise() [ 831.404692] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 831.404692] env[62736]: raise self.value [ 831.404692] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 831.404692] env[62736]: updated_port = self._update_port( [ 831.404692] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 831.404692] env[62736]: _ensure_no_port_binding_failure(port) [ 831.404692] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 831.404692] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 831.405944] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 831.405944] env[62736]: Removing descriptor: 17 [ 831.405944] env[62736]: ERROR nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] Traceback (most recent call last): [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] yield resources [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.driver.spawn(context, instance, image_meta, [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 831.405944] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] vm_ref = self.build_virtual_machine(instance, [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] vif_infos = vmwarevif.get_vif_info(self._session, [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] for vif in network_info: [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self._sync_wrapper(fn, *args, **kwargs) [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.wait() [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self[:] = self._gt.wait() [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self._exit_event.wait() [ 831.406517] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] result = hub.switch() [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self.greenlet.switch() [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] result = function(*args, **kwargs) [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return func(*args, **kwargs) [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise e [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] nwinfo = self.network_api.allocate_for_instance( [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 831.407107] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] created_port_ids = self._update_ports_for_instance( [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] with excutils.save_and_reraise_exception(): [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.force_reraise() [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise self.value [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] updated_port = self._update_port( [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] _ensure_no_port_binding_failure(port) [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 831.407696] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise exception.PortBindingFailed(port_id=port['id']) [ 831.408247] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 831.408247] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] [ 831.408247] env[62736]: INFO nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Terminating instance [ 831.408247] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquiring lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 831.587900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 37988933d77748b2a6e683d365c197b5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.595688] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37988933d77748b2a6e683d365c197b5 [ 831.603931] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 831.635010] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397746, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.43734} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 831.635365] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 831.635661] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 831.635907] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-29200c3f-0133-4a67-a279-077e77627524 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 831.642474] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 831.642474] env[62736]: value = "task-397747" [ 831.642474] env[62736]: _type = "Task" [ 831.642474] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 831.650270] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397747, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 831.675993] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 831.676620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg cae6ea1e51454d77ab7b8592b914837a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.685033] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cae6ea1e51454d77ab7b8592b914837a [ 831.730323] env[62736]: DEBUG nova.network.neutron [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 831.822182] env[62736]: DEBUG nova.network.neutron [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 831.822182] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] Expecting reply to msg 6fd731ef10bc4f8ca7673ed1081682ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 831.834588] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6fd731ef10bc4f8ca7673ed1081682ea [ 832.153996] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397747, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.061716} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 832.154266] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 832.155023] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-22a47f0b-388c-4f74-8457-8fbfdd445f49 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.176763] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Reconfiguring VM instance instance-0000003f to attach disk [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 832.179092] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-58f0109e-fd13-43df-8fb8-8d15d483473c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.193722] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Releasing lock "refresh_cache-99735498-3c85-4a01-bc05-4def014618a8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 832.193927] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 832.194099] env[62736]: DEBUG nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 832.194256] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 832.201647] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 832.201647] env[62736]: value = "task-397748" [ 832.201647] env[62736]: _type = "Task" [ 832.201647] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 832.209532] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 832.210072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg b4eb4be6c13e4e73a891afb1b49987a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.213771] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397748, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 832.219144] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4eb4be6c13e4e73a891afb1b49987a1 [ 832.332057] env[62736]: DEBUG oslo_concurrency.lockutils [req-c7db2bb7-f080-4f2e-a27c-93435220038d req-805a4122-d5b1-478c-a8f4-e3ece0fe7a83 service nova] Releasing lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 832.332695] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquired lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 832.332876] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 832.333296] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 5ba856daaee24867a67a521213fd2ea2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.339997] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ba856daaee24867a67a521213fd2ea2 [ 832.382759] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-018c875a-971b-4321-bc1c-d1413881761a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.390194] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68b98af9-3431-4792-93f6-965d5d6775a0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.422059] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ab31524-03c1-4be6-8644-a6ade0549a4a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.429360] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f11a1847-7dc4-4755-812e-c8496925d166 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.442726] env[62736]: DEBUG nova.compute.provider_tree [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 832.443414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 4dd13e96085d422d877c19daaa7df2d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.452093] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4dd13e96085d422d877c19daaa7df2d5 [ 832.711183] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397748, 'name': ReconfigVM_Task, 'duration_secs': 0.267588} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 832.711496] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Reconfigured VM instance instance-0000003f to attach disk [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 832.712097] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-051d7c4c-4e25-4dc2-94c9-e9d732891537 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 832.715168] env[62736]: DEBUG nova.network.neutron [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 832.715611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 234fadc96af549c9af2f283903b03f09 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.718459] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 832.718459] env[62736]: value = "task-397749" [ 832.718459] env[62736]: _type = "Task" [ 832.718459] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 832.725665] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397749, 'name': Rename_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 832.726134] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 234fadc96af549c9af2f283903b03f09 [ 832.851764] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 832.945322] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 832.945840] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 115d6759645a4fa7b8466351f9bacced in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.947508] env[62736]: DEBUG nova.scheduler.client.report [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 832.949973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ed617f0706af49f99b3dec3d3fb26abd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 832.955667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 115d6759645a4fa7b8466351f9bacced [ 832.962773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed617f0706af49f99b3dec3d3fb26abd [ 833.218348] env[62736]: INFO nova.compute.manager [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 99735498-3c85-4a01-bc05-4def014618a8] Took 1.02 seconds to deallocate network for instance. [ 833.220329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 37fb817a6f884e23aba35b11e95f745b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.230928] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397749, 'name': Rename_Task, 'duration_secs': 0.128042} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 833.231214] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 833.231441] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-94297383-d9bb-45e9-bfaa-a371b02e50c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.237787] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 833.237787] env[62736]: value = "task-397750" [ 833.237787] env[62736]: _type = "Task" [ 833.237787] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 833.245701] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397750, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 833.258320] env[62736]: DEBUG nova.compute.manager [req-1cb48b2b-c671-42d0-91d7-868a79863e28 req-b2371c2c-b273-49ba-905f-deb7fd30a863 service nova] [instance: ee85f964-db45-404c-9016-b618748ec63c] Received event network-vif-deleted-33c97501-12cc-4d90-b4eb-9d442da54929 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 833.262422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37fb817a6f884e23aba35b11e95f745b [ 833.451830] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Releasing lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 833.452379] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 833.452575] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 833.453406] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.375s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.453870] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 833.455453] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 755fffbbaa5e4318b3aa8deb9b4abe82 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.456494] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7c7750c2-418f-4308-ac23-5150e760ed35 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.478363] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.695s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.480194] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg bfa50954d21c402eb744dd4f25335990 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.487793] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-561c449a-e0ad-4d91-a889-0749bde7ccc5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 833.503464] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 755fffbbaa5e4318b3aa8deb9b4abe82 [ 833.514113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfa50954d21c402eb744dd4f25335990 [ 833.514755] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ee85f964-db45-404c-9016-b618748ec63c could not be found. [ 833.514957] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 833.515130] env[62736]: INFO nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Took 0.06 seconds to destroy the instance on the hypervisor. [ 833.515362] env[62736]: DEBUG oslo.service.loopingcall [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 833.515574] env[62736]: DEBUG nova.compute.manager [-] [instance: ee85f964-db45-404c-9016-b618748ec63c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 833.515667] env[62736]: DEBUG nova.network.neutron [-] [instance: ee85f964-db45-404c-9016-b618748ec63c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 833.529866] env[62736]: DEBUG nova.network.neutron [-] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 833.530325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a4124bbfb8d1402e902800dce8ab75d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.536115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4124bbfb8d1402e902800dce8ab75d9 [ 833.725773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 9e9faf83699d4411897fd15d2828a274 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.748542] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397750, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 833.769656] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e9faf83699d4411897fd15d2828a274 [ 833.980530] env[62736]: DEBUG nova.compute.utils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 833.981220] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg cd490f1f75594e69a2d4724dd3ac9c14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 833.982477] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 833.983714] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 833.994393] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd490f1f75594e69a2d4724dd3ac9c14 [ 834.033545] env[62736]: DEBUG nova.network.neutron [-] [instance: ee85f964-db45-404c-9016-b618748ec63c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 834.033545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8845128486c14fbb87309bec2ae9b7dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.041976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8845128486c14fbb87309bec2ae9b7dd [ 834.052289] env[62736]: DEBUG nova.policy [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 834.237865] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41cff82e-6b7a-46d4-a9c8-abdf2384b69c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.245251] env[62736]: INFO nova.scheduler.client.report [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Deleted allocations for instance 99735498-3c85-4a01-bc05-4def014618a8 [ 834.258419] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397750, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 834.259111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 384cfa449a8c449496cd56f69ddbf149 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.260757] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b373d280-bee0-4406-8bca-d99e4a2d8d3d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.292656] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 384cfa449a8c449496cd56f69ddbf149 [ 834.293657] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43e77950-80af-424f-9ea1-1f48df232367 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.301216] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e2da949-eaf5-4c9c-a356-756f173a8548 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.315710] env[62736]: DEBUG nova.compute.provider_tree [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 834.316245] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 5457b42324244f748417902924923c32 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.357917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5457b42324244f748417902924923c32 [ 834.434370] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Successfully created port: b4e7353c-b6e4-4257-a9f3-5b3989087fad {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 834.489204] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 834.491954] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 1679895e30ab4a05ab81de18f64dc732 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.523520] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1679895e30ab4a05ab81de18f64dc732 [ 834.536085] env[62736]: INFO nova.compute.manager [-] [instance: ee85f964-db45-404c-9016-b618748ec63c] Took 1.02 seconds to deallocate network for instance. [ 834.538216] env[62736]: DEBUG nova.compute.claims [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 834.538398] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 834.750649] env[62736]: DEBUG oslo_vmware.api [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397750, 'name': PowerOnVM_Task, 'duration_secs': 1.042869} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 834.750958] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 834.751195] env[62736]: INFO nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Took 5.87 seconds to spawn the instance on the hypervisor. [ 834.751412] env[62736]: DEBUG nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 834.752219] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7584518-4ab9-4da2-a487-263a5bb377e2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.759248] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 6bcdebd828704ea4beba2d218a4f89e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.771842] env[62736]: DEBUG oslo_concurrency.lockutils [None req-90a7f354-b213-4e50-b09e-e7191df4e099 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "99735498-3c85-4a01-bc05-4def014618a8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 137.818s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 834.774748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg e2d2e0e3b0ae4ce8a8dd8e5be48c2f43 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.787607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2d2e0e3b0ae4ce8a8dd8e5be48c2f43 [ 834.792595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bcdebd828704ea4beba2d218a4f89e0 [ 834.818866] env[62736]: DEBUG nova.scheduler.client.report [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 834.821483] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg a682f35ba31847dd8efb772dba004ec4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 834.831558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a682f35ba31847dd8efb772dba004ec4 [ 835.003292] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 427013e1088e49b293c6984d535531f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.050495] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 427013e1088e49b293c6984d535531f0 [ 835.282239] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 835.284021] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 56cd647bfb194166989f2610c78a8f17 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.285170] env[62736]: INFO nova.compute.manager [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Took 27.52 seconds to build instance. [ 835.285536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 14f7b301b51d4486b3b715c25f75240a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.306486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14f7b301b51d4486b3b715c25f75240a [ 835.317662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56cd647bfb194166989f2610c78a8f17 [ 835.324298] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.846s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.324901] env[62736]: ERROR nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Traceback (most recent call last): [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.driver.spawn(context, instance, image_meta, [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self._vmops.spawn(context, instance, image_meta, injected_files, [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] vm_ref = self.build_virtual_machine(instance, [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] vif_infos = vmwarevif.get_vif_info(self._session, [ 835.324901] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] for vif in network_info: [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self._sync_wrapper(fn, *args, **kwargs) [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.wait() [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self[:] = self._gt.wait() [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self._exit_event.wait() [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] result = hub.switch() [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 835.325213] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return self.greenlet.switch() [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] result = function(*args, **kwargs) [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] return func(*args, **kwargs) [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise e [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] nwinfo = self.network_api.allocate_for_instance( [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] created_port_ids = self._update_ports_for_instance( [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] with excutils.save_and_reraise_exception(): [ 835.325519] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] self.force_reraise() [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise self.value [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] updated_port = self._update_port( [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] _ensure_no_port_binding_failure(port) [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] raise exception.PortBindingFailed(port_id=port['id']) [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] nova.exception.PortBindingFailed: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. [ 835.325825] env[62736]: ERROR nova.compute.manager [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] [ 835.326090] env[62736]: DEBUG nova.compute.utils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 835.326708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.385s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.328665] env[62736]: INFO nova.compute.claims [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 835.330848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 27c3cfaf2b3f469e86f2fb9a8f1d3e6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.332477] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Build of instance 6c4ac72f-1636-4f4c-928c-0a6fe895ce37 was re-scheduled: Binding failed for port 902fe20e-4e9c-49b8-b7b0-c5b27cf5de23, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 835.333078] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 835.333363] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquiring lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 835.333602] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Acquired lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 835.333770] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 835.334183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 85f784005869457692df989893bcc551 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.341062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85f784005869457692df989893bcc551 [ 835.370085] env[62736]: DEBUG nova.compute.manager [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Received event network-changed-b4e7353c-b6e4-4257-a9f3-5b3989087fad {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 835.370085] env[62736]: DEBUG nova.compute.manager [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Refreshing instance network info cache due to event network-changed-b4e7353c-b6e4-4257-a9f3-5b3989087fad. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 835.370085] env[62736]: DEBUG oslo_concurrency.lockutils [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] Acquiring lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 835.370085] env[62736]: DEBUG oslo_concurrency.lockutils [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] Acquired lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 835.370085] env[62736]: DEBUG nova.network.neutron [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Refreshing network info cache for port b4e7353c-b6e4-4257-a9f3-5b3989087fad {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 835.370249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] Expecting reply to msg eb3c116a269043ce8151682893a76a45 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.370249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27c3cfaf2b3f469e86f2fb9a8f1d3e6c [ 835.373558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb3c116a269043ce8151682893a76a45 [ 835.393130] env[62736]: ERROR nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 835.393130] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.393130] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 835.393130] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 835.393130] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.393130] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.393130] env[62736]: ERROR nova.compute.manager raise self.value [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 835.393130] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 835.393130] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.393130] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 835.393656] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.393656] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 835.393656] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 835.393656] env[62736]: ERROR nova.compute.manager [ 835.393656] env[62736]: Traceback (most recent call last): [ 835.393656] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 835.393656] env[62736]: listener.cb(fileno) [ 835.393656] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 835.393656] env[62736]: result = function(*args, **kwargs) [ 835.393656] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 835.393656] env[62736]: return func(*args, **kwargs) [ 835.393656] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 835.393656] env[62736]: raise e [ 835.393656] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.393656] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 835.393656] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 835.393656] env[62736]: created_port_ids = self._update_ports_for_instance( [ 835.393656] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 835.393656] env[62736]: with excutils.save_and_reraise_exception(): [ 835.393656] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.393656] env[62736]: self.force_reraise() [ 835.393656] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.393656] env[62736]: raise self.value [ 835.393656] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 835.393656] env[62736]: updated_port = self._update_port( [ 835.393656] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.393656] env[62736]: _ensure_no_port_binding_failure(port) [ 835.393656] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.393656] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 835.394450] env[62736]: nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 835.394450] env[62736]: Removing descriptor: 17 [ 835.506244] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 835.530206] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 835.530471] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 835.530656] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 835.530842] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 835.530989] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 835.531133] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 835.531337] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 835.531499] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 835.531886] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 835.532075] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 835.532256] env[62736]: DEBUG nova.virt.hardware [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 835.533102] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21193801-6c08-4afb-a393-5eeefe5a79b3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.541137] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e67c704f-0791-4915-9172-796ccfa07fe4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.555864] env[62736]: ERROR nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Traceback (most recent call last): [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] yield resources [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.driver.spawn(context, instance, image_meta, [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self._vmops.spawn(context, instance, image_meta, injected_files, [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] vm_ref = self.build_virtual_machine(instance, [ 835.555864] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] vif_infos = vmwarevif.get_vif_info(self._session, [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] for vif in network_info: [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return self._sync_wrapper(fn, *args, **kwargs) [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.wait() [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self[:] = self._gt.wait() [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return self._exit_event.wait() [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 835.556271] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] current.throw(*self._exc) [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] result = function(*args, **kwargs) [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return func(*args, **kwargs) [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise e [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] nwinfo = self.network_api.allocate_for_instance( [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] created_port_ids = self._update_ports_for_instance( [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] with excutils.save_and_reraise_exception(): [ 835.556650] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.force_reraise() [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise self.value [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] updated_port = self._update_port( [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] _ensure_no_port_binding_failure(port) [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise exception.PortBindingFailed(port_id=port['id']) [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 835.557028] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] [ 835.557028] env[62736]: INFO nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Terminating instance [ 835.558142] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 835.794577] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8a9891a5-18b4-49d2-9051-d0466b6843da tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 121.320s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.795827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2eab2c2339994914ba9adca32caede52 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.805667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2eab2c2339994914ba9adca32caede52 [ 835.806856] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.818730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 5488e8f05d18466195afe6c76a2fb57f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.831357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5488e8f05d18466195afe6c76a2fb57f [ 835.834026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 6adf5097c3a44cee81feee1c57efda35 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.842056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6adf5097c3a44cee81feee1c57efda35 [ 835.849749] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 835.884427] env[62736]: DEBUG nova.network.neutron [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 835.939168] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 835.939772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 48dfdbc086b34139b16735d7ef0ac4a5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.946966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48dfdbc086b34139b16735d7ef0ac4a5 [ 835.957869] env[62736]: DEBUG nova.network.neutron [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 835.958341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] Expecting reply to msg 55727311ffde4a548b3cda4d0f48b4b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 835.965719] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55727311ffde4a548b3cda4d0f48b4b4 [ 836.297734] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 836.299587] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 231f91faab6e47408e3dfe402c983c58 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.321558] env[62736]: INFO nova.compute.manager [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Rebuilding instance [ 836.340421] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 231f91faab6e47408e3dfe402c983c58 [ 836.360094] env[62736]: DEBUG nova.compute.manager [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 836.361359] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19c98539-df50-4888-a88d-044225f53b0c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.369867] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 49463e42f84642c7adce52d69eff5b5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.398745] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49463e42f84642c7adce52d69eff5b5c [ 836.442295] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Releasing lock "refresh_cache-6c4ac72f-1636-4f4c-928c-0a6fe895ce37" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 836.442538] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 836.442734] env[62736]: DEBUG nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 836.442901] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 836.457875] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 836.458431] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 5cef981ad1984fafa8dda8089785a745 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.459901] env[62736]: DEBUG oslo_concurrency.lockutils [req-a2bda798-237d-4217-af5f-967b091dff83 req-a91ea3a2-ddf7-44b0-968d-64cbf126330c service nova] Releasing lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 836.460267] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 836.460469] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 836.460926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 50db86ddb56a4e628bced680c5a761bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.465170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5cef981ad1984fafa8dda8089785a745 [ 836.470769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50db86ddb56a4e628bced680c5a761bd [ 836.572119] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8783743a-0bb1-46d5-9a98-f42b7f2b22c8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.579343] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-948e5c15-a256-4148-a872-9050abe798f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.611459] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e02c2293-4397-44af-a288-e2b9cc651089 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.618633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bff868b1-dda0-4f6f-80ff-e32032d4c5ae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.631631] env[62736]: DEBUG nova.compute.provider_tree [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 836.632169] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg afd9035a1708460ca24e2fbc4085aa09 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.639145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afd9035a1708460ca24e2fbc4085aa09 [ 836.816472] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 836.873579] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 836.873915] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-4035768f-da8e-4efd-ae6d-0e8e05e39e6d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.882774] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 836.882774] env[62736]: value = "task-397751" [ 836.882774] env[62736]: _type = "Task" [ 836.882774] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 836.890861] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397751, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 836.960993] env[62736]: DEBUG nova.network.neutron [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 836.961523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 74ef93d7adf44186a92968d1486e2f3b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 836.969536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74ef93d7adf44186a92968d1486e2f3b [ 836.981472] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 837.051598] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 837.052147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 549cc6dfb41c45d6b8f96e7679956f78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.059735] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 549cc6dfb41c45d6b8f96e7679956f78 [ 837.135027] env[62736]: DEBUG nova.scheduler.client.report [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 837.137426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 40a4ad7e729443b296a4bdfc26741391 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.149894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40a4ad7e729443b296a4bdfc26741391 [ 837.391977] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397751, 'name': PowerOffVM_Task, 'duration_secs': 0.174416} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 837.392273] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 837.392487] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 837.393238] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a171900-70f6-4d17-be75-5920444a9f25 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.399393] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 837.400543] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0f1d3070-a037-4fa8-bdd7-59dcbae52d19 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.403215] env[62736]: DEBUG nova.compute.manager [req-67018514-8012-4ca7-ad2c-4cec196bfd72 req-2d58826f-1ad2-49c7-bc2a-79cba8d31862 service nova] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Received event network-vif-deleted-b4e7353c-b6e4-4257-a9f3-5b3989087fad {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 837.426263] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 837.426477] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 837.426786] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Deleting the datastore file [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 837.427093] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-614c7e99-5a83-4e9e-93aa-bc3ce928b326 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.432366] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 837.432366] env[62736]: value = "task-397753" [ 837.432366] env[62736]: _type = "Task" [ 837.432366] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 837.439679] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397753, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 837.464420] env[62736]: INFO nova.compute.manager [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] [instance: 6c4ac72f-1636-4f4c-928c-0a6fe895ce37] Took 1.02 seconds to deallocate network for instance. [ 837.466139] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 81018ac4832148c18bf9e23ab5e9cdfe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.499563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81018ac4832148c18bf9e23ab5e9cdfe [ 837.554677] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 837.555098] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 837.555278] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 837.555997] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e061d8bd-f403-492a-b5de-e15cc472188b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.564177] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f73b1b5-927e-4237-bcb4-b021d891fce7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.585954] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d7947374-0e8f-4d00-a430-923f0f2bb858 could not be found. [ 837.586202] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 837.588190] env[62736]: INFO nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Took 0.03 seconds to destroy the instance on the hypervisor. [ 837.588474] env[62736]: DEBUG oslo.service.loopingcall [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 837.588690] env[62736]: DEBUG nova.compute.manager [-] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 837.588787] env[62736]: DEBUG nova.network.neutron [-] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 837.603448] env[62736]: DEBUG nova.network.neutron [-] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 837.603952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0f943986d2284508a4ef08c133a707c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.610151] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f943986d2284508a4ef08c133a707c5 [ 837.640673] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.314s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.641201] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 837.642836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 5308057e03e54a0481e444fa6e5a91c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.643852] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.863s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.645315] env[62736]: INFO nova.compute.claims [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 837.646991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg ec0395de5aa9432c9b1c5d85f52f30ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.684816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5308057e03e54a0481e444fa6e5a91c9 [ 837.685396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec0395de5aa9432c9b1c5d85f52f30ce [ 837.942231] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397753, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.09841} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 837.942437] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 837.942622] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 837.942799] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 837.944557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg d1c78238ab674394913bb145b8e1a20e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.970523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg 196c734baf89401c9b346c08cd9c7df6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 837.975295] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1c78238ab674394913bb145b8e1a20e [ 837.999511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 196c734baf89401c9b346c08cd9c7df6 [ 838.106246] env[62736]: DEBUG nova.network.neutron [-] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 838.106709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 30c84b2b53534ea89ad6b48f2f532aac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.115193] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30c84b2b53534ea89ad6b48f2f532aac [ 838.150693] env[62736]: DEBUG nova.compute.utils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 838.151437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 2dfc3699416a43f5a81cb725aaf68896 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.153611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2eac71708c3f4694a067447adf57acbb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.154524] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 838.154697] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 838.162191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2eac71708c3f4694a067447adf57acbb [ 838.162651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dfc3699416a43f5a81cb725aaf68896 [ 838.193978] env[62736]: DEBUG nova.policy [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '09d6c42a5b174d3d8f4eab5ad842826f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '2cf54adae5eb4b4eb3f8cef36e2ff1e9', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 838.443915] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Successfully created port: f6facc53-17f2-461e-880c-00ea6c9b33c0 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 838.449121] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg ecfc085549c14446adb2c8f8282cdf55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.493396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ecfc085549c14446adb2c8f8282cdf55 [ 838.494690] env[62736]: INFO nova.scheduler.client.report [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Deleted allocations for instance 6c4ac72f-1636-4f4c-928c-0a6fe895ce37 [ 838.501677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Expecting reply to msg f234dcdbb8f44cfbbb89c3beeb1a3541 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.520042] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f234dcdbb8f44cfbbb89c3beeb1a3541 [ 838.609046] env[62736]: INFO nova.compute.manager [-] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Took 1.02 seconds to deallocate network for instance. [ 838.611596] env[62736]: DEBUG nova.compute.claims [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 838.611937] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 838.655962] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 838.657828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 6d868d794a734ce9968f7b5278572127 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.708830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d868d794a734ce9968f7b5278572127 [ 838.901146] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a24ddc2-8549-4548-90f6-0403944cb094 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.908793] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3ab18c7-66fa-4a3b-8b71-4c30474f9c26 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.939901] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0f23e31-13af-4c11-9e44-e5e240965484 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.947596] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae0db2b7-9ed8-4a36-8f6c-78169fee40b1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.960941] env[62736]: DEBUG nova.compute.provider_tree [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 838.961097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 638ed72313a94c4bb996b6c4ae4114b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 838.971367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 638ed72313a94c4bb996b6c4ae4114b3 [ 838.995781] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 838.995781] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 838.995781] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 838.995781] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 838.995995] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 838.995995] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 838.995995] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 838.995995] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 838.996193] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 838.996397] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 838.996637] env[62736]: DEBUG nova.virt.hardware [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 838.997607] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a824f59-03fa-4197-880b-f615d2af847f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.004441] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3394e696-4294-404c-886b-56dade178294 tempest-ServerRescueNegativeTestJSON-107795642 tempest-ServerRescueNegativeTestJSON-107795642-project-member] Lock "6c4ac72f-1636-4f4c-928c-0a6fe895ce37" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 141.546s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 839.005065] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 557baf381fb94444bbff5d9b7e690d6f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.007123] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9eb7d118-a1c9-4827-9bdf-1dfa9e1eabd8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.020579] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 839.026230] env[62736]: DEBUG oslo.service.loopingcall [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 839.026839] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 557baf381fb94444bbff5d9b7e690d6f [ 839.027260] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 839.027556] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-2ed6f25f-4373-465d-b31d-6ff8fb88be32 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.046715] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 839.046715] env[62736]: value = "task-397754" [ 839.046715] env[62736]: _type = "Task" [ 839.046715] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 839.055397] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397754, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 839.113127] env[62736]: DEBUG nova.compute.manager [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Received event network-changed-f6facc53-17f2-461e-880c-00ea6c9b33c0 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 839.113472] env[62736]: DEBUG nova.compute.manager [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Refreshing instance network info cache due to event network-changed-f6facc53-17f2-461e-880c-00ea6c9b33c0. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 839.113766] env[62736]: DEBUG oslo_concurrency.lockutils [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] Acquiring lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 839.113964] env[62736]: DEBUG oslo_concurrency.lockutils [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] Acquired lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 839.114161] env[62736]: DEBUG nova.network.neutron [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Refreshing network info cache for port f6facc53-17f2-461e-880c-00ea6c9b33c0 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 839.115104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] Expecting reply to msg 119e787bbe27426dbd5f8aac327dee5d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.123237] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 119e787bbe27426dbd5f8aac327dee5d [ 839.162593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg ad3e8fed348c453b98a933f6b4d4aa0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.200100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad3e8fed348c453b98a933f6b4d4aa0e [ 839.253755] env[62736]: ERROR nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 839.253755] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 839.253755] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 839.253755] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 839.253755] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 839.253755] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 839.253755] env[62736]: ERROR nova.compute.manager raise self.value [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 839.253755] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 839.253755] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 839.253755] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 839.254226] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 839.254226] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 839.254226] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 839.254226] env[62736]: ERROR nova.compute.manager [ 839.254226] env[62736]: Traceback (most recent call last): [ 839.254226] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 839.254226] env[62736]: listener.cb(fileno) [ 839.254226] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 839.254226] env[62736]: result = function(*args, **kwargs) [ 839.254226] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 839.254226] env[62736]: return func(*args, **kwargs) [ 839.254226] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 839.254226] env[62736]: raise e [ 839.254226] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 839.254226] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 839.254226] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 839.254226] env[62736]: created_port_ids = self._update_ports_for_instance( [ 839.254226] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 839.254226] env[62736]: with excutils.save_and_reraise_exception(): [ 839.254226] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 839.254226] env[62736]: self.force_reraise() [ 839.254226] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 839.254226] env[62736]: raise self.value [ 839.254226] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 839.254226] env[62736]: updated_port = self._update_port( [ 839.254226] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 839.254226] env[62736]: _ensure_no_port_binding_failure(port) [ 839.254226] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 839.254226] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 839.255064] env[62736]: nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 839.255064] env[62736]: Removing descriptor: 16 [ 839.463680] env[62736]: DEBUG nova.scheduler.client.report [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 839.466384] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1130c8d519174d4c8cd1549d76565bc3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.488161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1130c8d519174d4c8cd1549d76565bc3 [ 839.511309] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 839.513031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 087c226d541a429882d8610f565c4dd9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.541725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 087c226d541a429882d8610f565c4dd9 [ 839.556289] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397754, 'name': CreateVM_Task, 'duration_secs': 0.229209} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 839.556520] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 839.556957] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 839.557164] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 839.557512] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 839.557789] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3d3b3174-69ff-407c-b231-df28fc78e7eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.562475] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 839.562475] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52cd1464-0b8b-da03-c5bf-4d4cc2b1c1c4" [ 839.562475] env[62736]: _type = "Task" [ 839.562475] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 839.570197] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52cd1464-0b8b-da03-c5bf-4d4cc2b1c1c4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 839.632126] env[62736]: DEBUG nova.network.neutron [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 839.665266] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 839.687687] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 839.687989] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 839.688210] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 839.688436] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 839.688621] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 839.688803] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 839.689041] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 839.689234] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 839.689504] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 839.689726] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 839.689946] env[62736]: DEBUG nova.virt.hardware [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 839.690881] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f07a4da9-e961-4ab4-969a-e65ec3ee2992 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.700678] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb866e6a-164e-4056-9de3-5fb3078e531c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.715402] env[62736]: ERROR nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Traceback (most recent call last): [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] yield resources [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.driver.spawn(context, instance, image_meta, [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] vm_ref = self.build_virtual_machine(instance, [ 839.715402] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] vif_infos = vmwarevif.get_vif_info(self._session, [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] for vif in network_info: [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return self._sync_wrapper(fn, *args, **kwargs) [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.wait() [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self[:] = self._gt.wait() [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return self._exit_event.wait() [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 839.715710] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] current.throw(*self._exc) [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] result = function(*args, **kwargs) [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return func(*args, **kwargs) [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise e [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] nwinfo = self.network_api.allocate_for_instance( [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] created_port_ids = self._update_ports_for_instance( [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] with excutils.save_and_reraise_exception(): [ 839.716063] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.force_reraise() [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise self.value [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] updated_port = self._update_port( [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] _ensure_no_port_binding_failure(port) [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise exception.PortBindingFailed(port_id=port['id']) [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 839.716408] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] [ 839.716408] env[62736]: INFO nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Terminating instance [ 839.718715] env[62736]: DEBUG nova.network.neutron [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.719285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] Expecting reply to msg 569977dcdfa24f1b8838cfb8ff876557 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.720386] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquiring lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 839.727147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 569977dcdfa24f1b8838cfb8ff876557 [ 839.968980] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.325s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 839.969626] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 839.971773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg b28e6b4600204e0289e19c2ca6aa2d8a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 839.972827] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.995s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 839.982022] env[62736]: INFO nova.compute.claims [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 839.983581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 7cc475bcee0d4517bcc45a36eec32b04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 840.011076] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b28e6b4600204e0289e19c2ca6aa2d8a [ 840.014002] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7cc475bcee0d4517bcc45a36eec32b04 [ 840.032264] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.072850] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52cd1464-0b8b-da03-c5bf-4d4cc2b1c1c4, 'name': SearchDatastore_Task, 'duration_secs': 0.009127} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 840.073240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 840.073529] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 840.073841] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 840.074062] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 840.074290] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 840.074591] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8969038c-19b5-4d43-9bac-0e037b55e64b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 840.082389] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 840.082627] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 840.083353] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fd7f8d66-bec4-425c-ae65-0c8991dc87aa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 840.088291] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 840.088291] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52c7f8eb-d519-607d-f18c-92972107245d" [ 840.088291] env[62736]: _type = "Task" [ 840.088291] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 840.095336] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52c7f8eb-d519-607d-f18c-92972107245d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 840.221543] env[62736]: DEBUG oslo_concurrency.lockutils [req-2b769b6d-db9e-44cc-927c-ae276ea9fabc req-87713946-41ef-4e0b-b091-45a0f2f1eda9 service nova] Releasing lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 840.222305] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquired lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 840.222670] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 840.223188] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 10cebdab1fec46469db2840e45f9fb3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 840.229741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10cebdab1fec46469db2840e45f9fb3a [ 840.486879] env[62736]: DEBUG nova.compute.utils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 840.487568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1ebb3ce87e7d42e4ad2f18104ed36d7f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 840.489666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 685cf41ab6e24576845168e96196213e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 840.490907] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 840.490907] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 840.496379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 685cf41ab6e24576845168e96196213e [ 840.498211] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ebb3ce87e7d42e4ad2f18104ed36d7f [ 840.534547] env[62736]: DEBUG nova.policy [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 840.598355] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52c7f8eb-d519-607d-f18c-92972107245d, 'name': SearchDatastore_Task, 'duration_secs': 0.008187} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 840.599133] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-70a813c9-d93f-4274-9a4c-148ae9274261 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 840.603947] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 840.603947] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52b2985b-0c3d-8282-19d0-b655c9d08b22" [ 840.603947] env[62736]: _type = "Task" [ 840.603947] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 840.612400] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52b2985b-0c3d-8282-19d0-b655c9d08b22, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 840.762028] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 840.793817] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Successfully created port: 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 840.953144] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 840.953683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 51f026b970324fedba78faeff456819d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 840.963253] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51f026b970324fedba78faeff456819d [ 840.992035] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 840.994012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 56456a1117674c93b87739f4a0e560fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 841.043327] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56456a1117674c93b87739f4a0e560fc [ 841.114882] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52b2985b-0c3d-8282-19d0-b655c9d08b22, 'name': SearchDatastore_Task, 'duration_secs': 0.008514} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 841.115062] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 841.115321] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 841.115572] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a997cf03-712b-420e-ae53-adf25ff838f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.126051] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 841.126051] env[62736]: value = "task-397755" [ 841.126051] env[62736]: _type = "Task" [ 841.126051] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 841.134208] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397755, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 841.143761] env[62736]: DEBUG nova.compute.manager [req-ee0ebe94-fe9c-4e1a-9f23-e1e06ad23c91 req-d3b8250d-f480-44fa-9554-0f0a76df097f service nova] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Received event network-vif-deleted-f6facc53-17f2-461e-880c-00ea6c9b33c0 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 841.252706] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b06ec9b7-1530-4152-abf3-5deb3392a780 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.262901] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee53e9de-a8d0-4a71-b1a6-3b7eb03a389e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.297826] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55f34583-6a63-4e74-8c9e-f4d79113929c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.308696] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8434c4ff-87bc-4af9-bc3c-d112294c06da {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.325437] env[62736]: DEBUG nova.compute.provider_tree [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 841.326267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a1fab3a364e7455685abc775fbc3420c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 841.337680] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1fab3a364e7455685abc775fbc3420c [ 841.456241] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Releasing lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 841.456682] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 841.456883] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 841.457190] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-119dc30c-0272-4c51-b190-d5e9262710fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.480895] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34d80af9-6d0d-4681-ab7d-ce54f3edb2d3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.501664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 31bbcd18d1704139a323698a9fd5e54a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 841.509711] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe could not be found. [ 841.509964] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 841.510151] env[62736]: INFO nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Took 0.05 seconds to destroy the instance on the hypervisor. [ 841.510404] env[62736]: DEBUG oslo.service.loopingcall [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 841.510935] env[62736]: DEBUG nova.compute.manager [-] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 841.511034] env[62736]: DEBUG nova.network.neutron [-] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 841.536747] env[62736]: DEBUG nova.network.neutron [-] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 841.537316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6a385825851f4621b2b0a1b2076fce45 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 841.543748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31bbcd18d1704139a323698a9fd5e54a [ 841.545660] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a385825851f4621b2b0a1b2076fce45 [ 841.636552] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397755, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.435481} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 841.636833] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 841.637047] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 841.637302] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-d59b3288-f92b-47b3-8d33-b8d6356ded94 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 841.643004] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 841.643004] env[62736]: value = "task-397756" [ 841.643004] env[62736]: _type = "Task" [ 841.643004] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 841.650841] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397756, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 841.749495] env[62736]: ERROR nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 841.749495] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 841.749495] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 841.749495] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 841.749495] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 841.749495] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 841.749495] env[62736]: ERROR nova.compute.manager raise self.value [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 841.749495] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 841.749495] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 841.749495] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 841.749976] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 841.749976] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 841.749976] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 841.749976] env[62736]: ERROR nova.compute.manager [ 841.749976] env[62736]: Traceback (most recent call last): [ 841.749976] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 841.749976] env[62736]: listener.cb(fileno) [ 841.749976] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 841.749976] env[62736]: result = function(*args, **kwargs) [ 841.749976] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 841.749976] env[62736]: return func(*args, **kwargs) [ 841.749976] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 841.749976] env[62736]: raise e [ 841.749976] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 841.749976] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 841.749976] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 841.749976] env[62736]: created_port_ids = self._update_ports_for_instance( [ 841.749976] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 841.749976] env[62736]: with excutils.save_and_reraise_exception(): [ 841.749976] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 841.749976] env[62736]: self.force_reraise() [ 841.749976] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 841.749976] env[62736]: raise self.value [ 841.749976] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 841.749976] env[62736]: updated_port = self._update_port( [ 841.749976] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 841.749976] env[62736]: _ensure_no_port_binding_failure(port) [ 841.749976] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 841.749976] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 841.750727] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 841.750727] env[62736]: Removing descriptor: 16 [ 841.830601] env[62736]: DEBUG nova.scheduler.client.report [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 841.831842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 8edf1cfe3f78496383cc04d64dc971c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 841.846222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8edf1cfe3f78496383cc04d64dc971c9 [ 842.013110] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 842.040098] env[62736]: DEBUG nova.network.neutron [-] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 842.040286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 508b91f198a746d69b39b93151498c24 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.043641] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 842.043641] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 842.043641] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 842.043641] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 842.044854] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 842.044854] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 842.044854] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 842.044854] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 842.044854] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 842.045015] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 842.045015] env[62736]: DEBUG nova.virt.hardware [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 842.045487] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a533544-dc85-4123-ab10-eeb19d5dc0dd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.049646] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 508b91f198a746d69b39b93151498c24 [ 842.053759] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a35c5580-e19f-4df4-a403-ac382eec7cba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.068755] env[62736]: ERROR nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Traceback (most recent call last): [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] yield resources [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.driver.spawn(context, instance, image_meta, [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] vm_ref = self.build_virtual_machine(instance, [ 842.068755] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] vif_infos = vmwarevif.get_vif_info(self._session, [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] for vif in network_info: [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return self._sync_wrapper(fn, *args, **kwargs) [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.wait() [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self[:] = self._gt.wait() [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return self._exit_event.wait() [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 842.069115] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] current.throw(*self._exc) [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] result = function(*args, **kwargs) [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return func(*args, **kwargs) [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise e [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] nwinfo = self.network_api.allocate_for_instance( [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] created_port_ids = self._update_ports_for_instance( [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] with excutils.save_and_reraise_exception(): [ 842.069489] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.force_reraise() [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise self.value [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] updated_port = self._update_port( [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] _ensure_no_port_binding_failure(port) [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise exception.PortBindingFailed(port_id=port['id']) [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 842.069826] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] [ 842.069826] env[62736]: INFO nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Terminating instance [ 842.071127] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 842.071283] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 842.071445] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 842.071863] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0f85a31673834f65b677d77658eac152 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.078259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f85a31673834f65b677d77658eac152 [ 842.153368] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397756, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.057752} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 842.153655] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 842.154428] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bf82a17-5ae9-411c-a3b8-181d58dc01aa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.173623] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Reconfiguring VM instance instance-0000003f to attach disk [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 842.173912] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-ea4a04ff-18bd-4b27-a2e7-d7ed0a697401 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.192524] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 842.192524] env[62736]: value = "task-397757" [ 842.192524] env[62736]: _type = "Task" [ 842.192524] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 842.200109] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397757, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 842.334605] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.362s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 842.335170] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 842.337203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg c5ae7cb39904455e8d657ce4ff67bc53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.338416] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.594s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 842.340419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg c42fced7819646c4980c854b81f51ab2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.372203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5ae7cb39904455e8d657ce4ff67bc53 [ 842.378698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c42fced7819646c4980c854b81f51ab2 [ 842.548963] env[62736]: INFO nova.compute.manager [-] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Took 1.04 seconds to deallocate network for instance. [ 842.551391] env[62736]: DEBUG nova.compute.claims [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 842.551580] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 842.588760] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 842.667160] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 842.667743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2b07362a9ec847a8ae6553866b1386c3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.675928] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b07362a9ec847a8ae6553866b1386c3 [ 842.701901] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397757, 'name': ReconfigVM_Task, 'duration_secs': 0.258597} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 842.702152] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Reconfigured VM instance instance-0000003f to attach disk [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f/5d2465e7-e473-46e9-a8ce-cc58dcdbd28f.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 842.702731] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-917d3d37-2eda-49d5-b416-1a4bbe6d4caa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 842.708965] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 842.708965] env[62736]: value = "task-397758" [ 842.708965] env[62736]: _type = "Task" [ 842.708965] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 842.715861] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397758, 'name': Rename_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 842.843334] env[62736]: DEBUG nova.compute.utils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 842.844014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg da67253af5a04da3b36c5b96f7c472dc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 842.845014] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 842.845194] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 842.856094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da67253af5a04da3b36c5b96f7c472dc [ 842.893731] env[62736]: DEBUG nova.policy [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b785ade05de40c88f4bf3ef92e5a25f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '721d9e362be84a33aa7956177650d445', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 843.090099] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ebce076-3189-4fb1-be36-6ea96f71ed40 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.098388] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb8a1960-eb2d-4c56-b41f-cd9f0d26256c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.134172] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef42d5a3-9149-47bb-ac3e-5560ff83863a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.147150] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-83fbed4f-1109-47e7-954e-32c4971802b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.161745] env[62736]: DEBUG nova.compute.provider_tree [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 843.162340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg f60533ff211642ac87870a145a3350be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.171437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f60533ff211642ac87870a145a3350be [ 843.172087] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 843.172488] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 843.172678] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 843.173689] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Successfully created port: 205ec1e4-aa48-4292-9a63-d00ef691588a {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 843.175653] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5fd982d4-ef93-4343-a383-5a4e8b724efe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.182111] env[62736]: DEBUG nova.compute.manager [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Received event network-changed-0ce0a4e3-87d1-48d5-bd07-a94ee27198e8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 843.182295] env[62736]: DEBUG nova.compute.manager [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Refreshing instance network info cache due to event network-changed-0ce0a4e3-87d1-48d5-bd07-a94ee27198e8. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 843.182506] env[62736]: DEBUG oslo_concurrency.lockutils [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] Acquiring lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 843.182674] env[62736]: DEBUG oslo_concurrency.lockutils [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] Acquired lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 843.182897] env[62736]: DEBUG nova.network.neutron [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Refreshing network info cache for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 843.183413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] Expecting reply to msg be518fead5d244309c53c16bd58b8f6f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.188966] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4abfa237-577d-428e-9648-c0add9b97e0c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.198947] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be518fead5d244309c53c16bd58b8f6f [ 843.210297] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d866dcf0-c08b-4b12-be83-8de3e69c439f could not be found. [ 843.210494] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 843.210666] env[62736]: INFO nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 843.210889] env[62736]: DEBUG oslo.service.loopingcall [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 843.213742] env[62736]: DEBUG nova.compute.manager [-] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 843.213742] env[62736]: DEBUG nova.network.neutron [-] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 843.220129] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397758, 'name': Rename_Task, 'duration_secs': 0.125889} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 843.220392] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 843.220587] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-223717e2-b993-44ad-90f6-7078412328d6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.227029] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 843.227029] env[62736]: value = "task-397759" [ 843.227029] env[62736]: _type = "Task" [ 843.227029] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 843.231135] env[62736]: DEBUG nova.network.neutron [-] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 843.231580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f293f9f43eb14642b3217274aea1e59b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.237897] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397759, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 843.238565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f293f9f43eb14642b3217274aea1e59b [ 843.352554] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 843.352554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 04a80f25202d4ddbac14e3d65b859698 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.385001] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04a80f25202d4ddbac14e3d65b859698 [ 843.665975] env[62736]: DEBUG nova.scheduler.client.report [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 843.669578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 5763d7070f1e4a1b9aafcf891439df33 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.684889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5763d7070f1e4a1b9aafcf891439df33 [ 843.706090] env[62736]: DEBUG nova.network.neutron [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 843.739669] env[62736]: DEBUG nova.network.neutron [-] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 843.740116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a3edc5e02d574a53aa9ee974af2d742f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.740954] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397759, 'name': PowerOnVM_Task} progress is 66%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 843.750899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3edc5e02d574a53aa9ee974af2d742f [ 843.785884] env[62736]: DEBUG nova.network.neutron [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 843.786460] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] Expecting reply to msg c3e0451e168e447e899b0928c5ef3f1b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.797157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3e0451e168e447e899b0928c5ef3f1b [ 843.855993] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg b547f2524206414eaad6fb2edbb48609 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 843.891515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b547f2524206414eaad6fb2edbb48609 [ 843.926046] env[62736]: ERROR nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 843.926046] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 843.926046] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 843.926046] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 843.926046] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 843.926046] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 843.926046] env[62736]: ERROR nova.compute.manager raise self.value [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 843.926046] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 843.926046] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 843.926046] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 843.926751] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 843.926751] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 843.926751] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 843.926751] env[62736]: ERROR nova.compute.manager [ 843.926751] env[62736]: Traceback (most recent call last): [ 843.926751] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 843.926751] env[62736]: listener.cb(fileno) [ 843.926751] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 843.926751] env[62736]: result = function(*args, **kwargs) [ 843.926751] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 843.926751] env[62736]: return func(*args, **kwargs) [ 843.926751] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 843.926751] env[62736]: raise e [ 843.926751] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 843.926751] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 843.926751] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 843.926751] env[62736]: created_port_ids = self._update_ports_for_instance( [ 843.926751] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 843.926751] env[62736]: with excutils.save_and_reraise_exception(): [ 843.926751] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 843.926751] env[62736]: self.force_reraise() [ 843.926751] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 843.926751] env[62736]: raise self.value [ 843.926751] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 843.926751] env[62736]: updated_port = self._update_port( [ 843.926751] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 843.926751] env[62736]: _ensure_no_port_binding_failure(port) [ 843.926751] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 843.926751] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 843.927925] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 843.927925] env[62736]: Removing descriptor: 16 [ 844.172648] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.834s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.172918] env[62736]: ERROR nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Traceback (most recent call last): [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.driver.spawn(context, instance, image_meta, [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] vm_ref = self.build_virtual_machine(instance, [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] vif_infos = vmwarevif.get_vif_info(self._session, [ 844.172918] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] for vif in network_info: [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return self._sync_wrapper(fn, *args, **kwargs) [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.wait() [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self[:] = self._gt.wait() [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return self._exit_event.wait() [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] current.throw(*self._exc) [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 844.173222] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] result = function(*args, **kwargs) [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] return func(*args, **kwargs) [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise e [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] nwinfo = self.network_api.allocate_for_instance( [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] created_port_ids = self._update_ports_for_instance( [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] with excutils.save_and_reraise_exception(): [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] self.force_reraise() [ 844.173530] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise self.value [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] updated_port = self._update_port( [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] _ensure_no_port_binding_failure(port) [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] raise exception.PortBindingFailed(port_id=port['id']) [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] nova.exception.PortBindingFailed: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. [ 844.173903] env[62736]: ERROR nova.compute.manager [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] [ 844.173903] env[62736]: DEBUG nova.compute.utils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 844.174926] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.961s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.176433] env[62736]: INFO nova.compute.claims [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 844.180300] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 6c6910c03bf74a9395da55b07a8b014c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.180957] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Build of instance 451b9e70-f202-4a10-8b19-34b6167ef14d was re-scheduled: Binding failed for port 6b2abb82-a5d2-4157-8b93-2ef7bfb32a0b, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 844.182963] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 844.182963] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquiring lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 844.182963] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Acquired lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 844.182963] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 844.182963] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 4f136e44a6e94294a7a5c1d62ebe7ee1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.197765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f136e44a6e94294a7a5c1d62ebe7ee1 [ 844.216149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c6910c03bf74a9395da55b07a8b014c [ 844.240600] env[62736]: DEBUG oslo_vmware.api [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397759, 'name': PowerOnVM_Task, 'duration_secs': 0.589528} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 844.241139] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 844.241452] env[62736]: DEBUG nova.compute.manager [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 844.241971] env[62736]: INFO nova.compute.manager [-] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Took 1.03 seconds to deallocate network for instance. [ 844.242763] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-130a12a0-8651-487f-a30c-0992b613ca61 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.247190] env[62736]: DEBUG nova.compute.claims [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 844.247572] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.251429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 1ae67160664644f0ac985bc3ee298c28 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.278849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ae67160664644f0ac985bc3ee298c28 [ 844.288933] env[62736]: DEBUG oslo_concurrency.lockutils [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] Releasing lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 844.289383] env[62736]: DEBUG nova.compute.manager [req-84b06b92-13bd-469a-bfb0-b7995e694f83 req-2c6695cb-a54a-4742-8239-45c2c7bdeb7c service nova] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Received event network-vif-deleted-0ce0a4e3-87d1-48d5-bd07-a94ee27198e8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 844.359479] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 844.384988] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 844.385527] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 844.385821] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 844.386118] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 844.386392] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 844.386648] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 844.386960] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 844.387219] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 844.387561] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 844.387857] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 844.388169] env[62736]: DEBUG nova.virt.hardware [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 844.389103] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3deba0f8-b639-4371-b320-392ed3fcd2ed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.397319] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d6e6557-62d5-49be-9585-66d72c5d347f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 844.412995] env[62736]: ERROR nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Traceback (most recent call last): [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] yield resources [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.driver.spawn(context, instance, image_meta, [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] vm_ref = self.build_virtual_machine(instance, [ 844.412995] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] vif_infos = vmwarevif.get_vif_info(self._session, [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] for vif in network_info: [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return self._sync_wrapper(fn, *args, **kwargs) [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.wait() [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self[:] = self._gt.wait() [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return self._exit_event.wait() [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 844.413394] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] current.throw(*self._exc) [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] result = function(*args, **kwargs) [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return func(*args, **kwargs) [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise e [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] nwinfo = self.network_api.allocate_for_instance( [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] created_port_ids = self._update_ports_for_instance( [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] with excutils.save_and_reraise_exception(): [ 844.413829] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.force_reraise() [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise self.value [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] updated_port = self._update_port( [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] _ensure_no_port_binding_failure(port) [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise exception.PortBindingFailed(port_id=port['id']) [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 844.414211] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] [ 844.414980] env[62736]: INFO nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Terminating instance [ 844.418517] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 844.418825] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 844.419114] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 844.419780] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg eb703a0eb3694add9e92a7c8fa38d42e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.426616] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb703a0eb3694add9e92a7c8fa38d42e [ 844.478197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 8785cef885954ae5bc369ed05b1d22c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.490993] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8785cef885954ae5bc369ed05b1d22c7 [ 844.685950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 2cc9a5a6421f4d1890b9e567d1f144fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.694053] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2cc9a5a6421f4d1890b9e567d1f144fc [ 844.705339] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 844.758395] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.776461] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 844.776925] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 94314fbc1cda47a9b6de3f86f4e93c2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.785156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94314fbc1cda47a9b6de3f86f4e93c2d [ 844.937705] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 844.981718] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.981973] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.982176] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.982361] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.982525] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.984675] env[62736]: INFO nova.compute.manager [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Terminating instance [ 844.986288] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "refresh_cache-5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 844.986441] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquired lock "refresh_cache-5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 844.986602] env[62736]: DEBUG nova.network.neutron [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 844.987050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 43d2bf7a397c40daa76d016b236b2d5b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 844.993784] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43d2bf7a397c40daa76d016b236b2d5b [ 845.011607] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 845.012114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 216f2951e2fb49b68ef8905b86e64d1f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.020184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 216f2951e2fb49b68ef8905b86e64d1f [ 845.207016] env[62736]: DEBUG nova.compute.manager [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Received event network-changed-205ec1e4-aa48-4292-9a63-d00ef691588a {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 845.207016] env[62736]: DEBUG nova.compute.manager [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Refreshing instance network info cache due to event network-changed-205ec1e4-aa48-4292-9a63-d00ef691588a. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 845.207016] env[62736]: DEBUG oslo_concurrency.lockutils [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] Acquiring lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 845.278645] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Releasing lock "refresh_cache-451b9e70-f202-4a10-8b19-34b6167ef14d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 845.279084] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 845.279453] env[62736]: DEBUG nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 845.279788] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 845.295141] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 845.295141] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 931536f3150b437a95d295c8517fe9b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.303065] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 931536f3150b437a95d295c8517fe9b3 [ 845.407691] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76a916c1-1396-408b-895d-1181cd584c9b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.415294] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac72423e-f274-4574-987e-6c136b8ea60b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.445633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d3754dc-a7df-486b-9a1b-e0da193fe40e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.452306] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-610d5211-e1c3-444a-9c29-7f38470c59a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.464931] env[62736]: DEBUG nova.compute.provider_tree [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 845.465664] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 108793d4d64e4e8da2a10f4d25264815 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.473061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 108793d4d64e4e8da2a10f4d25264815 [ 845.503411] env[62736]: DEBUG nova.network.neutron [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 845.514465] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 845.515004] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 845.515333] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 845.515741] env[62736]: DEBUG oslo_concurrency.lockutils [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] Acquired lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 845.516075] env[62736]: DEBUG nova.network.neutron [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Refreshing network info cache for port 205ec1e4-aa48-4292-9a63-d00ef691588a {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 845.516606] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] Expecting reply to msg 5504b8e2a1864f5e86237ed3c6c40563 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.517537] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f98bac60-e99d-44ec-be70-a8841501888e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.524180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5504b8e2a1864f5e86237ed3c6c40563 [ 845.527177] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8f12dee-1910-4f84-afbe-724b503d4d51 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 845.548316] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ddcf62db-af40-49d2-9465-66a10f0f2e2d could not be found. [ 845.548757] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 845.549085] env[62736]: INFO nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 845.549489] env[62736]: DEBUG oslo.service.loopingcall [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 845.549831] env[62736]: DEBUG nova.compute.manager [-] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 845.550049] env[62736]: DEBUG nova.network.neutron [-] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 845.552427] env[62736]: DEBUG nova.network.neutron [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 845.553030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg ed0b5d750b214cf38df995cd5fdd3a0d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.560972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed0b5d750b214cf38df995cd5fdd3a0d [ 845.567077] env[62736]: DEBUG nova.network.neutron [-] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 845.567667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 773478ae28a54e3a93576a15a1a2c1ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.579206] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 773478ae28a54e3a93576a15a1a2c1ae [ 845.796277] env[62736]: DEBUG nova.network.neutron [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 845.797153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 01d0a88b13ae4476af6898a279d9a3bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.805052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01d0a88b13ae4476af6898a279d9a3bd [ 845.969531] env[62736]: DEBUG nova.scheduler.client.report [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 845.972325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg b07f0ad4891e4c858fae4e0f6a2cf451 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 845.984220] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b07f0ad4891e4c858fae4e0f6a2cf451 [ 846.036963] env[62736]: DEBUG nova.network.neutron [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 846.055620] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Releasing lock "refresh_cache-5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 846.056354] env[62736]: DEBUG nova.compute.manager [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 846.056754] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 846.057759] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e0569c4-ba05-4c0a-b9c1-f919046ebb64 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.065533] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 846.065914] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-02f083ff-4e54-4a35-ae3f-ca4ccd201d17 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.069915] env[62736]: DEBUG nova.network.neutron [-] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 846.070465] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bb794089d4c54a77993b0b383dd55f2c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.072973] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 846.072973] env[62736]: value = "task-397760" [ 846.072973] env[62736]: _type = "Task" [ 846.072973] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 846.082604] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397760, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 846.089504] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bb794089d4c54a77993b0b383dd55f2c [ 846.161538] env[62736]: DEBUG nova.network.neutron [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 846.162286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] Expecting reply to msg e7e6142a03e0462b832978a646456495 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.170783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7e6142a03e0462b832978a646456495 [ 846.300111] env[62736]: INFO nova.compute.manager [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] [instance: 451b9e70-f202-4a10-8b19-34b6167ef14d] Took 1.02 seconds to deallocate network for instance. [ 846.302054] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 2793a89ffefd4df99d2a25794d41c81d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.336783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2793a89ffefd4df99d2a25794d41c81d [ 846.475324] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.300s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.476198] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 846.478183] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 1514e54aa3104080878c4c0951d4b7fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.479431] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.941s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.481271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg ad712775c34046f1a73fd893f9b794e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.514242] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1514e54aa3104080878c4c0951d4b7fa [ 846.515282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad712775c34046f1a73fd893f9b794e7 [ 846.574694] env[62736]: INFO nova.compute.manager [-] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Took 1.02 seconds to deallocate network for instance. [ 846.580335] env[62736]: DEBUG nova.compute.claims [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 846.580674] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.586359] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397760, 'name': PowerOffVM_Task, 'duration_secs': 0.177809} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 846.586711] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 846.587025] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 846.587405] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-8a8ba373-e05d-4e12-a067-b6a7126f1bc4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.609542] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 846.609938] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 846.610241] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Deleting the datastore file [datastore1] 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 846.610663] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d321b6a6-921a-49a0-ad99-7f0c8c4c5f4f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 846.617892] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for the task: (returnval){ [ 846.617892] env[62736]: value = "task-397762" [ 846.617892] env[62736]: _type = "Task" [ 846.617892] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 846.626384] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397762, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 846.664918] env[62736]: DEBUG oslo_concurrency.lockutils [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] Releasing lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 846.665513] env[62736]: DEBUG nova.compute.manager [req-cc60833f-3b01-4c88-b403-a8416357adc2 req-2fbb6f6e-9c3e-431b-b249-fa732fa25b99 service nova] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Received event network-vif-deleted-205ec1e4-aa48-4292-9a63-d00ef691588a {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 846.806309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 167beb33d2c246b4bf327d6b1efff105 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.835355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 167beb33d2c246b4bf327d6b1efff105 [ 846.984647] env[62736]: DEBUG nova.compute.utils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 846.985591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 158b07ee72cb490a8f69ca25e2abc4cb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 846.987178] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 846.987502] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 846.996299] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 158b07ee72cb490a8f69ca25e2abc4cb [ 847.027725] env[62736]: DEBUG nova.policy [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9825cb6e55f8419299e0b4f909b722d6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e049414f9871493398a8e5a07c3ef938', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 847.128661] env[62736]: DEBUG oslo_vmware.api [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Task: {'id': task-397762, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.099654} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 847.128918] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 847.129096] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 847.129266] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 847.129459] env[62736]: INFO nova.compute.manager [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Took 1.07 seconds to destroy the instance on the hypervisor. [ 847.129702] env[62736]: DEBUG oslo.service.loopingcall [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 847.129883] env[62736]: DEBUG nova.compute.manager [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 847.129974] env[62736]: DEBUG nova.network.neutron [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 847.157030] env[62736]: DEBUG nova.network.neutron [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 847.157783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0337c00219974a5d8dc93dfa757543e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.165355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0337c00219974a5d8dc93dfa757543e7 [ 847.223893] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9c4526cc-e569-4ca6-803d-35353dde5903 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.241907] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c083b4f7-8fa7-451e-90bc-9d54d332eb73 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.279882] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f35847e3-9442-491d-a819-e045a3c72e8f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.287126] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ad5a67c-cb83-4a8f-a33e-90cd20f819fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 847.300257] env[62736]: DEBUG nova.compute.provider_tree [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 847.300785] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg ea128d6f07b048fdbb4591fbb1bb7eed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.309063] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea128d6f07b048fdbb4591fbb1bb7eed [ 847.328947] env[62736]: INFO nova.scheduler.client.report [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Deleted allocations for instance 451b9e70-f202-4a10-8b19-34b6167ef14d [ 847.336112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Expecting reply to msg 919d1e09a6cf444ba332751ba000794b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.348367] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 919d1e09a6cf444ba332751ba000794b [ 847.468908] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Successfully created port: 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 847.490932] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 847.493637] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 16833e6337d54eb18a25405c188d3d1c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.521357] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 847.521653] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 847.522298] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 7d8ea06f286f434e9e205b06067a6cc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.532850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16833e6337d54eb18a25405c188d3d1c [ 847.535336] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d8ea06f286f434e9e205b06067a6cc6 [ 847.659150] env[62736]: DEBUG nova.network.neutron [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 847.659740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c95a07bdbf414c4b8157d21c1087b34a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.668010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c95a07bdbf414c4b8157d21c1087b34a [ 847.803822] env[62736]: DEBUG nova.scheduler.client.report [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 847.806404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 9ea9a18c78914e76b08594cc79520546 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.818917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ea9a18c78914e76b08594cc79520546 [ 847.838090] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c3fe70d3-6af1-4cb6-aa74-8d5614569729 tempest-SecurityGroupsTestJSON-257580290 tempest-SecurityGroupsTestJSON-257580290-project-member] Lock "451b9e70-f202-4a10-8b19-34b6167ef14d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 137.513s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 847.838716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg a6f2e86fbd6e4d269309d234a7dd9c96 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 847.849514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6f2e86fbd6e4d269309d234a7dd9c96 [ 847.998985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 57ba9d26f02b4cdcb3b3b992b0062d2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.032622] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.032716] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 848.032799] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 848.033345] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 6f43834e3887494e859ed27cdcff5a1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.034371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57ba9d26f02b4cdcb3b3b992b0062d2d [ 848.046636] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f43834e3887494e859ed27cdcff5a1d [ 848.162401] env[62736]: INFO nova.compute.manager [-] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Took 1.03 seconds to deallocate network for instance. [ 848.166371] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 082939a4d0a243c29856cf4404e49694 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.194513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 082939a4d0a243c29856cf4404e49694 [ 848.308720] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.829s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.309408] env[62736]: ERROR nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] Traceback (most recent call last): [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.driver.spawn(context, instance, image_meta, [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self._vmops.spawn(context, instance, image_meta, injected_files, [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] vm_ref = self.build_virtual_machine(instance, [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] vif_infos = vmwarevif.get_vif_info(self._session, [ 848.309408] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] for vif in network_info: [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self._sync_wrapper(fn, *args, **kwargs) [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.wait() [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self[:] = self._gt.wait() [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self._exit_event.wait() [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] result = hub.switch() [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 848.309754] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return self.greenlet.switch() [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] result = function(*args, **kwargs) [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] return func(*args, **kwargs) [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise e [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] nwinfo = self.network_api.allocate_for_instance( [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] created_port_ids = self._update_ports_for_instance( [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] with excutils.save_and_reraise_exception(): [ 848.310132] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] self.force_reraise() [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise self.value [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] updated_port = self._update_port( [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] _ensure_no_port_binding_failure(port) [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] raise exception.PortBindingFailed(port_id=port['id']) [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] nova.exception.PortBindingFailed: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. [ 848.310516] env[62736]: ERROR nova.compute.manager [instance: ee85f964-db45-404c-9016-b618748ec63c] [ 848.310828] env[62736]: DEBUG nova.compute.utils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 848.311632] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 12.505s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.313007] env[62736]: INFO nova.compute.claims [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 848.314563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 831228e17e9b4138b1279e8c90539458 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.315733] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Build of instance ee85f964-db45-404c-9016-b618748ec63c was re-scheduled: Binding failed for port 33c97501-12cc-4d90-b4eb-9d442da54929, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 848.316390] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 848.316635] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquiring lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 848.316783] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Acquired lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 848.316941] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 848.317288] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 750d8db6d07b435cb96a9a9e5dff9036 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.325396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 750d8db6d07b435cb96a9a9e5dff9036 [ 848.341023] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 848.342684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg e5bae92997e441a19808be1c036bd9ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.366761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 831228e17e9b4138b1279e8c90539458 [ 848.381988] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5bae92997e441a19808be1c036bd9ab [ 848.398688] env[62736]: DEBUG nova.compute.manager [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Received event network-changed-08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 848.398688] env[62736]: DEBUG nova.compute.manager [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Refreshing instance network info cache due to event network-changed-08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 848.398688] env[62736]: DEBUG oslo_concurrency.lockutils [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] Acquiring lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 848.398688] env[62736]: DEBUG oslo_concurrency.lockutils [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] Acquired lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 848.398688] env[62736]: DEBUG nova.network.neutron [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Refreshing network info cache for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 848.398866] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] Expecting reply to msg c137c3c728d9474887985ac71915c603 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.406134] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c137c3c728d9474887985ac71915c603 [ 848.506182] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 848.528925] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 848.529454] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 848.529764] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 848.530088] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 848.530372] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 848.530663] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 848.530998] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 848.531293] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 848.531595] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 848.531887] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 848.532222] env[62736]: DEBUG nova.virt.hardware [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 848.533527] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8b465a99-c748-4f07-b886-810414fe84fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.538780] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Skipping network cache update for instance because it is being deleted. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9943}} [ 848.539064] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.539319] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.539593] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.539858] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.540152] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 848.540414] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 848.540728] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.541421] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.542012] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.542422] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.542792] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.545231] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.545507] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 848.545863] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 848.546325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg e10a61b138a74c8ba525d5eee75d469a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.548812] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e145f47-6c18-4be6-aea7-57728bd5b402 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 848.564725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e10a61b138a74c8ba525d5eee75d469a [ 848.670039] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.677020] env[62736]: ERROR nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 848.677020] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 848.677020] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 848.677020] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 848.677020] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 848.677020] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 848.677020] env[62736]: ERROR nova.compute.manager raise self.value [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 848.677020] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 848.677020] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 848.677020] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 848.677566] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 848.677566] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 848.677566] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 848.677566] env[62736]: ERROR nova.compute.manager [ 848.677974] env[62736]: Traceback (most recent call last): [ 848.678060] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 848.678060] env[62736]: listener.cb(fileno) [ 848.678132] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 848.678132] env[62736]: result = function(*args, **kwargs) [ 848.678209] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 848.678209] env[62736]: return func(*args, **kwargs) [ 848.678278] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 848.678278] env[62736]: raise e [ 848.678348] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 848.678348] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 848.678785] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 848.678785] env[62736]: created_port_ids = self._update_ports_for_instance( [ 848.678893] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 848.678893] env[62736]: with excutils.save_and_reraise_exception(): [ 848.678961] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 848.678961] env[62736]: self.force_reraise() [ 848.679027] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 848.679027] env[62736]: raise self.value [ 848.679093] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 848.679093] env[62736]: updated_port = self._update_port( [ 848.679167] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 848.679167] env[62736]: _ensure_no_port_binding_failure(port) [ 848.679319] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 848.679319] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 848.679417] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 848.679469] env[62736]: Removing descriptor: 17 [ 848.680228] env[62736]: ERROR nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Traceback (most recent call last): [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] yield resources [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.driver.spawn(context, instance, image_meta, [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] vm_ref = self.build_virtual_machine(instance, [ 848.680228] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] vif_infos = vmwarevif.get_vif_info(self._session, [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] for vif in network_info: [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self._sync_wrapper(fn, *args, **kwargs) [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.wait() [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self[:] = self._gt.wait() [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self._exit_event.wait() [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 848.680625] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] result = hub.switch() [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self.greenlet.switch() [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] result = function(*args, **kwargs) [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return func(*args, **kwargs) [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise e [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] nwinfo = self.network_api.allocate_for_instance( [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] created_port_ids = self._update_ports_for_instance( [ 848.680993] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] with excutils.save_and_reraise_exception(): [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.force_reraise() [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise self.value [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] updated_port = self._update_port( [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] _ensure_no_port_binding_failure(port) [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise exception.PortBindingFailed(port_id=port['id']) [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 848.681364] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] [ 848.682047] env[62736]: INFO nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Terminating instance [ 848.684146] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 848.820850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 677af4c4e9c7483bbdac56c0dd524a55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 848.829340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 677af4c4e9c7483bbdac56c0dd524a55 [ 848.860281] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 848.864127] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.913549] env[62736]: DEBUG nova.network.neutron [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 849.005723] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 849.006259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 2b563fb87db84f178aeef6d97bfe286f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 849.009153] env[62736]: DEBUG nova.network.neutron [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 849.009153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] Expecting reply to msg a3789474884740f2b686285ca27c2618 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 849.017972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b563fb87db84f178aeef6d97bfe286f [ 849.018533] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3789474884740f2b686285ca27c2618 [ 849.048907] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 849.508485] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Releasing lock "refresh_cache-ee85f964-db45-404c-9016-b618748ec63c" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 849.508722] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 849.508898] env[62736]: DEBUG nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 849.509058] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 849.518306] env[62736]: DEBUG oslo_concurrency.lockutils [req-a113604e-c46f-489a-87e0-3fbcb6f294fb req-903df7c0-f941-4717-b57a-17dafce8531d service nova] Releasing lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 849.520890] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquired lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 849.521069] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 849.521496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg f29142a69f3148739d51cccdc686d033 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 849.523455] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 849.524330] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 9b181ac0f9314fc2b467e8f3ed8a4b23 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 849.532031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f29142a69f3148739d51cccdc686d033 [ 849.532594] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b181ac0f9314fc2b467e8f3ed8a4b23 [ 849.576122] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38685357-3b19-4920-8fda-478976356eaf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.588518] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d0f8990-6bf6-4a59-858d-4bc332e525cc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.624996] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc234d2a-ec63-4fa6-87f9-244589ff83d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.632737] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4d6c7de-e19e-41f7-9c75-d53156c245e9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 849.647221] env[62736]: DEBUG nova.compute.provider_tree [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 849.647878] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg f5e33afde77b49c79d2eb9edad253a21 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 849.655032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5e33afde77b49c79d2eb9edad253a21 [ 850.032515] env[62736]: DEBUG nova.network.neutron [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 850.033030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 1b8afec883ec48adbc198d214eadaa9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.039805] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 850.043136] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1b8afec883ec48adbc198d214eadaa9f [ 850.151051] env[62736]: DEBUG nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 850.153565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 5fa820a7f13f46edb8db0f4db89b008a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.166570] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fa820a7f13f46edb8db0f4db89b008a [ 850.251150] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 850.251678] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 340f585d695f4f6a92ff0003e0cfd97c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.260236] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 340f585d695f4f6a92ff0003e0cfd97c [ 850.425318] env[62736]: DEBUG nova.compute.manager [req-4e63ec26-5203-4821-96e1-1fafb58e4e96 req-b7d9363d-0a50-4071-8532-7ba72e94d313 service nova] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Received event network-vif-deleted-08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 850.536086] env[62736]: INFO nova.compute.manager [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] [instance: ee85f964-db45-404c-9016-b618748ec63c] Took 1.03 seconds to deallocate network for instance. [ 850.537876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 95e2ccc5c85647a6857a537ffb4f0144 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.574758] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95e2ccc5c85647a6857a537ffb4f0144 [ 850.656372] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.345s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.656863] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 850.658621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 1c78d9858fd045aebb6f2a32b9e8cc8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.660227] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.844s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.661707] env[62736]: INFO nova.compute.claims [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 850.663238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 6e0134dcddbb45d5bd7d6d5038c3ae32 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.687260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c78d9858fd045aebb6f2a32b9e8cc8b [ 850.703674] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e0134dcddbb45d5bd7d6d5038c3ae32 [ 850.754357] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Releasing lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 850.754808] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 850.754995] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 850.755291] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6e9e37c0-265a-42cb-90a4-029906783581 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.764582] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99c35f7c-d509-47bc-a7e0-815e7fa6c819 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 850.788711] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb could not be found. [ 850.788975] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 850.789187] env[62736]: INFO nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Took 0.03 seconds to destroy the instance on the hypervisor. [ 850.789526] env[62736]: DEBUG oslo.service.loopingcall [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 850.789809] env[62736]: DEBUG nova.compute.manager [-] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 850.789932] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 850.811120] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 850.811895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5b3d883410e84beb99d36ee692687333 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 850.820158] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b3d883410e84beb99d36ee692687333 [ 851.043676] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg a1fbff99a4e34ee78594f3df2be7ba76 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.082083] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1fbff99a4e34ee78594f3df2be7ba76 [ 851.162265] env[62736]: DEBUG nova.compute.utils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 851.162906] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 47e763b277f64db9ab74b1bc4884d660 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.163856] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 851.164055] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 851.168125] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg a281a1aab0714692bde6828f38305e88 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.173189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47e763b277f64db9ab74b1bc4884d660 [ 851.177816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a281a1aab0714692bde6828f38305e88 [ 851.213225] env[62736]: DEBUG nova.policy [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce8f935dca594cfa87b680e92f1c492c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a34141edbd4482d9f709ad211179a90', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 851.314313] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 851.314917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 932786f58a934abca79486eba56232a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.323440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 932786f58a934abca79486eba56232a6 [ 851.506339] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Successfully created port: 6d2239ac-5dcb-4286-baf3-2cb4fd85180d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 851.581050] env[62736]: INFO nova.scheduler.client.report [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Deleted allocations for instance ee85f964-db45-404c-9016-b618748ec63c [ 851.586907] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Expecting reply to msg 01469c35ffc1467eab821398631507ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.607682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01469c35ffc1467eab821398631507ef [ 851.667471] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 851.669227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 7b417a8388ea448996744c9244fc4f91 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.705825] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b417a8388ea448996744c9244fc4f91 [ 851.818989] env[62736]: INFO nova.compute.manager [-] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Took 1.03 seconds to deallocate network for instance. [ 851.821270] env[62736]: DEBUG nova.compute.claims [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 851.821451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 851.901561] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dee67000-9516-411a-8e39-28146ec730a4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.909451] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1faae0a3-1468-4d57-b84c-cce0fc942cd6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.938257] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6cdcfb1f-de4f-4449-8e3a-62fdf271dadf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.945997] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-828db120-0aff-4bc4-8181-d5c97ce7858a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 851.960345] env[62736]: DEBUG nova.compute.provider_tree [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 851.960848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 728c0afe5d1c4f35ae0854a994886add in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.967673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 728c0afe5d1c4f35ae0854a994886add [ 851.982376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 86ebc179e51e4a1e8c9327c10061c06e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 851.992185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86ebc179e51e4a1e8c9327c10061c06e [ 852.093322] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a2af6359-9dfe-4f11-bc59-a146ba7d4955 tempest-ServerMetadataTestJSON-1364403100 tempest-ServerMetadataTestJSON-1364403100-project-member] Lock "ee85f964-db45-404c-9016-b618748ec63c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 127.429s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 852.094017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 768bb930b2194215800b8c2a6aa62ef5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.106604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 768bb930b2194215800b8c2a6aa62ef5 [ 852.176634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg c55192913cd143c4b5e8ffd23746cc6d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.217329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c55192913cd143c4b5e8ffd23746cc6d [ 852.305351] env[62736]: DEBUG nova.compute.manager [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Received event network-changed-6d2239ac-5dcb-4286-baf3-2cb4fd85180d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 852.305566] env[62736]: DEBUG nova.compute.manager [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Refreshing instance network info cache due to event network-changed-6d2239ac-5dcb-4286-baf3-2cb4fd85180d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 852.305787] env[62736]: DEBUG oslo_concurrency.lockutils [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] Acquiring lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 852.305929] env[62736]: DEBUG oslo_concurrency.lockutils [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] Acquired lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 852.306086] env[62736]: DEBUG nova.network.neutron [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Refreshing network info cache for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 852.306517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] Expecting reply to msg f3065d624eb6438d8d1eb00f295d796e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.316265] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3065d624eb6438d8d1eb00f295d796e [ 852.466502] env[62736]: DEBUG nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 852.469017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 0b599abf683344e1a32b55e2554c13f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.485243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b599abf683344e1a32b55e2554c13f9 [ 852.595751] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 852.597519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg f5ca3c182c4e428989c0d4587ee62c81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.606781] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 852.606781] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 852.606781] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 852.606781] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 852.606781] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 852.606781] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 852.606781] env[62736]: ERROR nova.compute.manager raise self.value [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 852.606781] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 852.606781] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 852.606781] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 852.607229] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 852.607229] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 852.607229] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 852.607229] env[62736]: ERROR nova.compute.manager [ 852.607229] env[62736]: Traceback (most recent call last): [ 852.607229] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 852.607229] env[62736]: listener.cb(fileno) [ 852.607229] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 852.607229] env[62736]: result = function(*args, **kwargs) [ 852.607229] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 852.607229] env[62736]: return func(*args, **kwargs) [ 852.607229] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 852.607229] env[62736]: raise e [ 852.607229] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 852.607229] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 852.607229] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 852.607229] env[62736]: created_port_ids = self._update_ports_for_instance( [ 852.607229] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 852.607229] env[62736]: with excutils.save_and_reraise_exception(): [ 852.607229] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 852.607229] env[62736]: self.force_reraise() [ 852.607229] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 852.607229] env[62736]: raise self.value [ 852.607229] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 852.607229] env[62736]: updated_port = self._update_port( [ 852.607229] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 852.607229] env[62736]: _ensure_no_port_binding_failure(port) [ 852.607229] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 852.607229] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 852.607900] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 852.607900] env[62736]: Removing descriptor: 16 [ 852.632140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5ca3c182c4e428989c0d4587ee62c81 [ 852.679928] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 852.705944] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 852.706195] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 852.706337] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 852.706515] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 852.706753] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 852.706916] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 852.707120] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 852.707273] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 852.707574] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 852.707759] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 852.707932] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 852.709212] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2de0684a-8676-4e48-a43b-a07b6c7ae9b9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.718036] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55788375-49c9-4411-9bf8-992dbdac50b8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 852.732263] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Traceback (most recent call last): [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] yield resources [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.driver.spawn(context, instance, image_meta, [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] vm_ref = self.build_virtual_machine(instance, [ 852.732263] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] vif_infos = vmwarevif.get_vif_info(self._session, [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] for vif in network_info: [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return self._sync_wrapper(fn, *args, **kwargs) [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.wait() [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self[:] = self._gt.wait() [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return self._exit_event.wait() [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 852.732668] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] current.throw(*self._exc) [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] result = function(*args, **kwargs) [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return func(*args, **kwargs) [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise e [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] nwinfo = self.network_api.allocate_for_instance( [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] created_port_ids = self._update_ports_for_instance( [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] with excutils.save_and_reraise_exception(): [ 852.733059] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.force_reraise() [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise self.value [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] updated_port = self._update_port( [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] _ensure_no_port_binding_failure(port) [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise exception.PortBindingFailed(port_id=port['id']) [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 852.733444] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] [ 852.733444] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Terminating instance [ 852.734560] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 852.829775] env[62736]: DEBUG nova.network.neutron [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 852.966791] env[62736]: DEBUG nova.network.neutron [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 852.967317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] Expecting reply to msg 37fc1bbba6244cb8a912cd66a447f0b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.974271] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.314s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 852.974840] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 852.976508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg a4dcbb41a9434c44979276a932e2fa9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 852.987770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37fc1bbba6244cb8a912cd66a447f0b4 [ 852.988216] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.376s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 852.990047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a19398b48f8a4ef19ef7a9c917213d98 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 853.017623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4dcbb41a9434c44979276a932e2fa9a [ 853.031770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a19398b48f8a4ef19ef7a9c917213d98 [ 853.118674] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.469611] env[62736]: DEBUG oslo_concurrency.lockutils [req-4c936469-2605-4d8e-b049-2770f2ee2ad3 req-af866e2d-edb7-4bfe-8dd9-08e387c2db4d service nova] Releasing lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 853.470048] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 853.470236] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 853.470997] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d11d8b37c36d4175ae9c95693d171241 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 853.479876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d11d8b37c36d4175ae9c95693d171241 [ 853.493387] env[62736]: DEBUG nova.compute.utils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 853.494046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg bf8229ca72e3489b9017c7d109cd64ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 853.512561] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 853.512561] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 853.514484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf8229ca72e3489b9017c7d109cd64ef [ 853.565387] env[62736]: DEBUG nova.policy [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ce8f935dca594cfa87b680e92f1c492c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '6a34141edbd4482d9f709ad211179a90', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 853.724235] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45e264b5-06b8-465a-9e5f-6d004c87105f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.731751] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fab7d6f-75ab-4dc1-8e5b-a788500f6bf3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.767573] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7936336-c131-4988-8ded-87873b4de988 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.776202] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73df9342-d424-4574-8a08-b0ddb133b514 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 853.790873] env[62736]: DEBUG nova.compute.provider_tree [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 853.791378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 308af2d3147d441cbc067dffc222707c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 853.798669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 308af2d3147d441cbc067dffc222707c [ 853.844201] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Successfully created port: 8892036c-2583-407d-8dd1-b44a5d64495f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 853.998643] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 854.008082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 07e07e068d7c45eab0454d5a50761e2a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.013880] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 854.050827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 07e07e068d7c45eab0454d5a50761e2a [ 854.204250] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 854.204810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 827cfb29467f41f6872d2ed24fcecaed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.213457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 827cfb29467f41f6872d2ed24fcecaed [ 854.299334] env[62736]: DEBUG nova.scheduler.client.report [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 854.301837] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d746dfdf59f541fd8a951b9c53cec511 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.316597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d746dfdf59f541fd8a951b9c53cec511 [ 854.347937] env[62736]: DEBUG nova.compute.manager [req-a5d6d65b-70c4-4a8f-93b4-ccf634fbd7d8 req-1d41e601-de43-447d-9ab6-df4a8f3026b0 service nova] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Received event network-vif-deleted-6d2239ac-5dcb-4286-baf3-2cb4fd85180d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 854.516291] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg f83ff0d9a3b44124a630fda5d1b5908d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.549829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f83ff0d9a3b44124a630fda5d1b5908d [ 854.697462] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 854.697462] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 854.697462] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 854.697462] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 854.697462] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 854.697462] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 854.697462] env[62736]: ERROR nova.compute.manager raise self.value [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 854.697462] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 854.697462] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 854.697462] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 854.697961] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 854.697961] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 854.697961] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 854.697961] env[62736]: ERROR nova.compute.manager [ 854.697961] env[62736]: Traceback (most recent call last): [ 854.697961] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 854.697961] env[62736]: listener.cb(fileno) [ 854.697961] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 854.697961] env[62736]: result = function(*args, **kwargs) [ 854.697961] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 854.697961] env[62736]: return func(*args, **kwargs) [ 854.697961] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 854.697961] env[62736]: raise e [ 854.697961] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 854.697961] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 854.697961] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 854.697961] env[62736]: created_port_ids = self._update_ports_for_instance( [ 854.697961] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 854.697961] env[62736]: with excutils.save_and_reraise_exception(): [ 854.697961] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 854.697961] env[62736]: self.force_reraise() [ 854.697961] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 854.697961] env[62736]: raise self.value [ 854.697961] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 854.697961] env[62736]: updated_port = self._update_port( [ 854.697961] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 854.697961] env[62736]: _ensure_no_port_binding_failure(port) [ 854.697961] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 854.697961] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 854.698762] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 854.698762] env[62736]: Removing descriptor: 16 [ 854.712403] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 854.712833] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 854.713085] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 854.713401] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-864f3b5a-397f-4ee2-9c6f-5c0fe5f9af3a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.724090] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d614a9b8-725a-4cc5-8669-59839ce6172c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 854.747298] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e could not be found. [ 854.747298] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 854.747298] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Took 0.03 seconds to destroy the instance on the hypervisor. [ 854.747298] env[62736]: DEBUG oslo.service.loopingcall [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 854.747298] env[62736]: DEBUG nova.compute.manager [-] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 854.747298] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 854.761522] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 854.762197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 37b6c76e8679410b8c0ff19dc872ffbd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.774979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37b6c76e8679410b8c0ff19dc872ffbd [ 854.804451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.816s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 854.805118] env[62736]: ERROR nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Traceback (most recent call last): [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.driver.spawn(context, instance, image_meta, [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self._vmops.spawn(context, instance, image_meta, injected_files, [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] vm_ref = self.build_virtual_machine(instance, [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] vif_infos = vmwarevif.get_vif_info(self._session, [ 854.805118] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] for vif in network_info: [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return self._sync_wrapper(fn, *args, **kwargs) [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.wait() [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self[:] = self._gt.wait() [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return self._exit_event.wait() [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] current.throw(*self._exc) [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 854.805465] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] result = function(*args, **kwargs) [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] return func(*args, **kwargs) [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise e [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] nwinfo = self.network_api.allocate_for_instance( [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] created_port_ids = self._update_ports_for_instance( [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] with excutils.save_and_reraise_exception(): [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] self.force_reraise() [ 854.805790] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise self.value [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] updated_port = self._update_port( [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] _ensure_no_port_binding_failure(port) [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] raise exception.PortBindingFailed(port_id=port['id']) [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] nova.exception.PortBindingFailed: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. [ 854.806116] env[62736]: ERROR nova.compute.manager [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] [ 854.806116] env[62736]: DEBUG nova.compute.utils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 854.807190] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 14.775s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 854.808705] env[62736]: INFO nova.compute.claims [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 854.810429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 27099f8be5c94ab49af0b83effa5ff15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.816169] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Build of instance d7947374-0e8f-4d00-a430-923f0f2bb858 was re-scheduled: Binding failed for port b4e7353c-b6e4-4257-a9f3-5b3989087fad, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 854.816169] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 854.816169] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 854.816169] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 854.816614] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 854.816614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 90717078c10947c5abc4ba6ac977cc6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 854.821449] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90717078c10947c5abc4ba6ac977cc6c [ 854.865961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27099f8be5c94ab49af0b83effa5ff15 [ 855.022607] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 855.050606] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 855.050879] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 855.051050] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 855.051260] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 855.051419] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 855.051597] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 855.051839] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 855.052072] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 855.052275] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 855.052457] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 855.052634] env[62736]: DEBUG nova.virt.hardware [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 855.053491] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-946558c4-0d2e-4509-98dc-ad37fd0533c5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.062042] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d6efc3a-395d-4d3f-bd20-bbffe6b6d3ff {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 855.080013] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Traceback (most recent call last): [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] yield resources [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.driver.spawn(context, instance, image_meta, [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self._vmops.spawn(context, instance, image_meta, injected_files, [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] vm_ref = self.build_virtual_machine(instance, [ 855.080013] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] vif_infos = vmwarevif.get_vif_info(self._session, [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] for vif in network_info: [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return self._sync_wrapper(fn, *args, **kwargs) [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.wait() [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self[:] = self._gt.wait() [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return self._exit_event.wait() [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 855.080379] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] current.throw(*self._exc) [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] result = function(*args, **kwargs) [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return func(*args, **kwargs) [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise e [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] nwinfo = self.network_api.allocate_for_instance( [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] created_port_ids = self._update_ports_for_instance( [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] with excutils.save_and_reraise_exception(): [ 855.080797] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.force_reraise() [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise self.value [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] updated_port = self._update_port( [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] _ensure_no_port_binding_failure(port) [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise exception.PortBindingFailed(port_id=port['id']) [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 855.081133] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] [ 855.081133] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Terminating instance [ 855.087964] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 855.088261] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 855.088436] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 855.088875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 62fa7513754241348bb1ae94b5ff1b22 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.096480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62fa7513754241348bb1ae94b5ff1b22 [ 855.269424] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.269926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg dc9c0590a4ec4c16b8c2bff08651701a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.279209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc9c0590a4ec4c16b8c2bff08651701a [ 855.319512] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 80a80e9a2eb2469392dd8a0f14e99eb4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.328978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80a80e9a2eb2469392dd8a0f14e99eb4 [ 855.336530] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.436472] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.437007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c02ecc2c7b75454490f512aff28efdc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.444439] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c02ecc2c7b75454490f512aff28efdc8 [ 855.610692] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.692204] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 855.692716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2f6b2952b5da4ba2ae7ae0f6cf7f1c66 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.700221] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f6b2952b5da4ba2ae7ae0f6cf7f1c66 [ 855.772437] env[62736]: INFO nova.compute.manager [-] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Took 1.03 seconds to deallocate network for instance. [ 855.774766] env[62736]: DEBUG nova.compute.claims [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 855.774947] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.939001] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-d7947374-0e8f-4d00-a430-923f0f2bb858" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 855.939349] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 855.939751] env[62736]: DEBUG nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 855.939911] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 855.961085] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.961675] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg fd8483459d6b4e378e0146b8c8506015 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 855.968593] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd8483459d6b4e378e0146b8c8506015 [ 856.081859] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-819d63c1-a287-42c2-a6f8-cc63828de7e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.090792] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b6a4b91-de58-4456-a2e5-33bcfba1bedd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.128169] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-600f6d55-548a-4828-ae28-cfd28ff68760 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.135496] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbee79ea-5830-4bec-bb40-14a21aa0f15c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.148956] env[62736]: DEBUG nova.compute.provider_tree [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 856.149435] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 2d1dd797c0a8432cb8f660e8a66659b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.162206] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d1dd797c0a8432cb8f660e8a66659b0 [ 856.195605] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 856.196072] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 856.196231] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 856.196563] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-19a1f035-e039-4b1d-8dde-e95aa62f3b48 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.205509] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c163680d-f589-4b85-8ffe-c60be7882adf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 856.226349] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1f90a84e-6c32-4846-8908-128323e33e98 could not be found. [ 856.226571] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 856.226735] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Took 0.03 seconds to destroy the instance on the hypervisor. [ 856.226971] env[62736]: DEBUG oslo.service.loopingcall [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 856.227182] env[62736]: DEBUG nova.compute.manager [-] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 856.227274] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 856.273898] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 856.274438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 35d4c9b7cc834eaea9949fd4117d4868 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.281302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35d4c9b7cc834eaea9949fd4117d4868 [ 856.373781] env[62736]: DEBUG nova.compute.manager [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Received event network-changed-8892036c-2583-407d-8dd1-b44a5d64495f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 856.374051] env[62736]: DEBUG nova.compute.manager [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Refreshing instance network info cache due to event network-changed-8892036c-2583-407d-8dd1-b44a5d64495f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 856.374373] env[62736]: DEBUG oslo_concurrency.lockutils [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] Acquiring lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 856.374564] env[62736]: DEBUG oslo_concurrency.lockutils [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] Acquired lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 856.374902] env[62736]: DEBUG nova.network.neutron [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Refreshing network info cache for port 8892036c-2583-407d-8dd1-b44a5d64495f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 856.375476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] Expecting reply to msg 7f0ccfe3c24f4bf8b8ee6644993b705b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.384402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f0ccfe3c24f4bf8b8ee6644993b705b [ 856.464102] env[62736]: DEBUG nova.network.neutron [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.464949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a64a3261d1e14ac5a0031353f32ce773 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.479341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a64a3261d1e14ac5a0031353f32ce773 [ 856.653666] env[62736]: DEBUG nova.scheduler.client.report [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 856.654895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg a4b4066718c8442f8387b3c62540c269 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.667010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4b4066718c8442f8387b3c62540c269 [ 856.736028] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "768cf3b2-227f-463a-b192-79c50874cded" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 856.736256] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "768cf3b2-227f-463a-b192-79c50874cded" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 856.775996] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.776457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3fd2d05ea6e94ea8843dcfef6df68385 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.784297] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fd2d05ea6e94ea8843dcfef6df68385 [ 856.900387] env[62736]: DEBUG nova.network.neutron [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 856.967599] env[62736]: INFO nova.compute.manager [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: d7947374-0e8f-4d00-a430-923f0f2bb858] Took 1.03 seconds to deallocate network for instance. [ 856.969175] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 0d3c21abfe4f4a36871746f7d7f85026 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.978261] env[62736]: DEBUG nova.network.neutron [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 856.978697] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] Expecting reply to msg 9fc53c6786c24c7985ec2f7c02a5aee3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 856.991852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fc53c6786c24c7985ec2f7c02a5aee3 [ 857.005306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d3c21abfe4f4a36871746f7d7f85026 [ 857.157223] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.350s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 857.157801] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 857.159427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 33dee1ff0e9f4e0f8cf6ff3126e9477a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 857.160498] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.609s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 857.162175] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 2cbae412d60b4f7b85755fbdc2d1a6cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 857.192269] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33dee1ff0e9f4e0f8cf6ff3126e9477a [ 857.195094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2cbae412d60b4f7b85755fbdc2d1a6cf [ 857.278751] env[62736]: INFO nova.compute.manager [-] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Took 1.05 seconds to deallocate network for instance. [ 857.281219] env[62736]: DEBUG nova.compute.claims [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 857.281385] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 857.473910] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 2183443613be4b5683e7e56c44587f81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 857.481761] env[62736]: DEBUG oslo_concurrency.lockutils [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] Releasing lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 857.482019] env[62736]: DEBUG nova.compute.manager [req-68efb7e1-4455-4857-86e0-ae9c1f9cf1ea req-20e7dee4-98e8-4975-9924-8b0060e84702 service nova] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Received event network-vif-deleted-8892036c-2583-407d-8dd1-b44a5d64495f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 857.506096] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2183443613be4b5683e7e56c44587f81 [ 857.665459] env[62736]: DEBUG nova.compute.utils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 857.666120] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 06e33efaffa44033b20500651cc1d315 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 857.670117] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 857.670284] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 857.676515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06e33efaffa44033b20500651cc1d315 [ 857.725972] env[62736]: DEBUG nova.policy [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '613209c7c103464ea473f32bcf0116b1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '7c343033abe74ef28f09e614ffad9254', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 857.893781] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e519deb-a2e5-4481-9c44-c9dd57ece64d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.901662] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41cb8567-25aa-450c-91d5-cadf4189e156 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.932728] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f3c9ff8-a1a2-41da-9159-1cffb2fb2f66 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.940095] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c702837c-4c99-4a6d-b568-394262f61e8e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 857.953090] env[62736]: DEBUG nova.compute.provider_tree [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 857.953605] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 3f74464fbbc54776945ed348320494d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 857.961424] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f74464fbbc54776945ed348320494d7 [ 858.003835] env[62736]: INFO nova.scheduler.client.report [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance d7947374-0e8f-4d00-a430-923f0f2bb858 [ 858.011388] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg fd82924ed3f14977a5b452cfe32948d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.024807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd82924ed3f14977a5b452cfe32948d5 [ 858.027424] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Successfully created port: 1dd6ec9b-97da-4ae2-a996-2fe67cf26328 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 858.173572] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 858.175569] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 9557835182424ad6bf57745a7dca5cc6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.219935] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9557835182424ad6bf57745a7dca5cc6 [ 858.304636] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Successfully created port: b31f2b6a-89c9-42d2-a751-e3e34066069c {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 858.462306] env[62736]: DEBUG nova.scheduler.client.report [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 858.464777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 3a79b75d63784f328543aa4049c42822 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.477810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a79b75d63784f328543aa4049c42822 [ 858.517284] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3d4299ab-4e1b-46b6-a286-83dfe570cd7f tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "d7947374-0e8f-4d00-a430-923f0f2bb858" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 130.596s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 858.517877] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 90ac395b64f9420a95021a16813679c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.527262] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90ac395b64f9420a95021a16813679c5 [ 858.680978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg e003166a1f3f483c86610f0da24954ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.717557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e003166a1f3f483c86610f0da24954ea [ 858.972037] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.811s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 858.972676] env[62736]: ERROR nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Traceback (most recent call last): [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.driver.spawn(context, instance, image_meta, [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] vm_ref = self.build_virtual_machine(instance, [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] vif_infos = vmwarevif.get_vif_info(self._session, [ 858.972676] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] for vif in network_info: [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return self._sync_wrapper(fn, *args, **kwargs) [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.wait() [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self[:] = self._gt.wait() [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return self._exit_event.wait() [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] current.throw(*self._exc) [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 858.973017] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] result = function(*args, **kwargs) [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] return func(*args, **kwargs) [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise e [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] nwinfo = self.network_api.allocate_for_instance( [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] created_port_ids = self._update_ports_for_instance( [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] with excutils.save_and_reraise_exception(): [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] self.force_reraise() [ 858.973333] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise self.value [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] updated_port = self._update_port( [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] _ensure_no_port_binding_failure(port) [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] raise exception.PortBindingFailed(port_id=port['id']) [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] nova.exception.PortBindingFailed: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. [ 858.973647] env[62736]: ERROR nova.compute.manager [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] [ 858.973647] env[62736]: DEBUG nova.compute.utils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 858.974583] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.727s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 858.976732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 03d47dd2b51548e1a055fee4d22234d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.978018] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Build of instance 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe was re-scheduled: Binding failed for port f6facc53-17f2-461e-880c-00ea6c9b33c0, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 858.978517] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 858.978701] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquiring lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 858.978842] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Acquired lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 858.978992] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 858.979357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg d70d2bafc0814cfcb32478f1a8f24c07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 858.986808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d70d2bafc0814cfcb32478f1a8f24c07 [ 859.010963] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03d47dd2b51548e1a055fee4d22234d6 [ 859.020498] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 859.022160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 42ab452c534d4454859700d92c5d0ee7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 859.051355] env[62736]: DEBUG nova.compute.manager [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Received event network-changed-1dd6ec9b-97da-4ae2-a996-2fe67cf26328 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 859.051355] env[62736]: DEBUG nova.compute.manager [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Refreshing instance network info cache due to event network-changed-1dd6ec9b-97da-4ae2-a996-2fe67cf26328. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 859.051561] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] Acquiring lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 859.051696] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] Acquired lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 859.051858] env[62736]: DEBUG nova.network.neutron [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Refreshing network info cache for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 859.053094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] Expecting reply to msg 390281af1dc84454a80138304891b98a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 859.059216] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42ab452c534d4454859700d92c5d0ee7 [ 859.059700] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 390281af1dc84454a80138304891b98a [ 859.187994] env[62736]: ERROR nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 859.187994] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 859.187994] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 859.187994] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 859.187994] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 859.187994] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 859.187994] env[62736]: ERROR nova.compute.manager raise self.value [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 859.187994] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 859.187994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 859.187994] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 859.188527] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 859.188527] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 859.188527] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 859.188527] env[62736]: ERROR nova.compute.manager [ 859.188527] env[62736]: Traceback (most recent call last): [ 859.188527] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 859.188527] env[62736]: listener.cb(fileno) [ 859.188527] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 859.188527] env[62736]: result = function(*args, **kwargs) [ 859.188527] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 859.188527] env[62736]: return func(*args, **kwargs) [ 859.188527] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 859.188527] env[62736]: raise e [ 859.188527] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 859.188527] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 859.188527] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 859.188527] env[62736]: created_port_ids = self._update_ports_for_instance( [ 859.188527] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 859.188527] env[62736]: with excutils.save_and_reraise_exception(): [ 859.188527] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 859.188527] env[62736]: self.force_reraise() [ 859.188527] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 859.188527] env[62736]: raise self.value [ 859.188527] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 859.188527] env[62736]: updated_port = self._update_port( [ 859.188527] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 859.188527] env[62736]: _ensure_no_port_binding_failure(port) [ 859.188527] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 859.188527] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 859.189286] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 859.189286] env[62736]: Removing descriptor: 16 [ 859.193214] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 859.212508] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 859.212731] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 859.212879] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 859.213051] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 859.213186] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 859.213320] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 859.213544] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 859.213697] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 859.213854] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 859.214002] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 859.214165] env[62736]: DEBUG nova.virt.hardware [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 859.214998] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-550fccac-1700-4225-b997-a03a260c11f0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.223152] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b57bd48-f337-49f5-97a0-04c598739bef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.247614] env[62736]: ERROR nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Traceback (most recent call last): [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] yield resources [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.driver.spawn(context, instance, image_meta, [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self._vmops.spawn(context, instance, image_meta, injected_files, [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] vm_ref = self.build_virtual_machine(instance, [ 859.247614] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] vif_infos = vmwarevif.get_vif_info(self._session, [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] for vif in network_info: [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return self._sync_wrapper(fn, *args, **kwargs) [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.wait() [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self[:] = self._gt.wait() [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return self._exit_event.wait() [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 859.248042] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] current.throw(*self._exc) [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] result = function(*args, **kwargs) [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return func(*args, **kwargs) [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise e [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] nwinfo = self.network_api.allocate_for_instance( [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] created_port_ids = self._update_ports_for_instance( [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] with excutils.save_and_reraise_exception(): [ 859.248408] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.force_reraise() [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise self.value [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] updated_port = self._update_port( [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] _ensure_no_port_binding_failure(port) [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise exception.PortBindingFailed(port_id=port['id']) [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 859.248780] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] [ 859.248780] env[62736]: INFO nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Terminating instance [ 859.249994] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 859.505767] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.549568] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 859.589770] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.590306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 5766d701fe7f4b68a5fff3bd3d0b3555 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 859.600345] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5766d701fe7f4b68a5fff3bd3d0b3555 [ 859.625695] env[62736]: DEBUG nova.network.neutron [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.762074] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0750883-280f-4245-80f2-83b26c3a6867 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.765658] env[62736]: DEBUG nova.network.neutron [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.766137] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] Expecting reply to msg 8badf7f6f1b249cba8622e44c8235e85 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 859.770337] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72115f0c-1ee0-4c6a-929c-0ff10ae3ba63 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.774290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8badf7f6f1b249cba8622e44c8235e85 [ 859.800537] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bd0e0ee-378b-475d-9587-4be02aa62a23 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.810908] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b70de00d-2303-409f-bd7e-899b5b5a5a22 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 859.824201] env[62736]: DEBUG nova.compute.provider_tree [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 859.824724] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 39921ffd05984fa5b623e25b9320268e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 859.832488] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39921ffd05984fa5b623e25b9320268e [ 860.093460] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Releasing lock "refresh_cache-76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 860.093703] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 860.093882] env[62736]: DEBUG nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 860.094047] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 860.108683] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 860.109291] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 55bcf5f3d2c74bcc8ff42b525ae29bcd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.117148] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55bcf5f3d2c74bcc8ff42b525ae29bcd [ 860.140147] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "1f1dd01d-006d-4569-8684-e1eb913ba5d4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 860.140371] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "1f1dd01d-006d-4569-8684-e1eb913ba5d4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 860.270775] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d696b26-cdea-4d84-b15e-f87d163b4348 req-d1317237-93fc-4c85-ac9e-4ff576e05fe6 service nova] Releasing lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 860.271513] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquired lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 860.271724] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 860.272165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 177bd611691a47fabd9d3f42fe417d70 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.278987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 177bd611691a47fabd9d3f42fe417d70 [ 860.327970] env[62736]: DEBUG nova.scheduler.client.report [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 860.330415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 28d7ad909be146ed8e050d8768ee7fd5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.343197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28d7ad909be146ed8e050d8768ee7fd5 [ 860.611194] env[62736]: DEBUG nova.network.neutron [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.611753] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 8fd6b1b4ede147fdb58e8d27a09734f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.619896] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8fd6b1b4ede147fdb58e8d27a09734f0 [ 860.807193] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 860.833241] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.859s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 860.833876] env[62736]: ERROR nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Traceback (most recent call last): [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.driver.spawn(context, instance, image_meta, [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] vm_ref = self.build_virtual_machine(instance, [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] vif_infos = vmwarevif.get_vif_info(self._session, [ 860.833876] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] for vif in network_info: [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return self._sync_wrapper(fn, *args, **kwargs) [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.wait() [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self[:] = self._gt.wait() [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return self._exit_event.wait() [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] current.throw(*self._exc) [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 860.834225] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] result = function(*args, **kwargs) [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] return func(*args, **kwargs) [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise e [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] nwinfo = self.network_api.allocate_for_instance( [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] created_port_ids = self._update_ports_for_instance( [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] with excutils.save_and_reraise_exception(): [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] self.force_reraise() [ 860.834535] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise self.value [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] updated_port = self._update_port( [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] _ensure_no_port_binding_failure(port) [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] raise exception.PortBindingFailed(port_id=port['id']) [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] nova.exception.PortBindingFailed: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. [ 860.834843] env[62736]: ERROR nova.compute.manager [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] [ 860.834843] env[62736]: DEBUG nova.compute.utils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 860.835778] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 16.077s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 860.835948] env[62736]: DEBUG nova.objects.instance [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] [instance: 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62736) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 860.837489] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg c537a71bf0d04e849d9e8fc1cf1847db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.839232] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Build of instance d866dcf0-c08b-4b12-be83-8de3e69c439f was re-scheduled: Binding failed for port 0ce0a4e3-87d1-48d5-bd07-a94ee27198e8, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 860.839688] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 860.839954] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 860.840089] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 860.840249] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 860.840646] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2f91b4809e504bf5a8d972db9a2f1349 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.848727] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2f91b4809e504bf5a8d972db9a2f1349 [ 860.867481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c537a71bf0d04e849d9e8fc1cf1847db [ 860.923525] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 860.924192] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 64305171e54c44b38f202e4967cbf436 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 860.931283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64305171e54c44b38f202e4967cbf436 [ 861.076218] env[62736]: DEBUG nova.compute.manager [req-22915316-a375-43a9-bf1f-0b3e55f87d07 req-9d58213a-c06a-44fc-abdf-f4e3d517ab7f service nova] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Received event network-vif-deleted-1dd6ec9b-97da-4ae2-a996-2fe67cf26328 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 861.114499] env[62736]: INFO nova.compute.manager [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] [instance: 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe] Took 1.02 seconds to deallocate network for instance. [ 861.116974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 69dcd5de6d5c4396b8c47832f8a30051 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.148537] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69dcd5de6d5c4396b8c47832f8a30051 [ 861.344270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 6ccbb2077cb54cae94dc3061b6088757 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.351425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ccbb2077cb54cae94dc3061b6088757 [ 861.364798] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.426019] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Releasing lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 861.426451] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 861.426636] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 861.426927] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-3004f941-0e4e-4650-bb3c-aa8c566edc1d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.435210] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdb6c005-bfc5-4f6e-97b4-c9acfe4c5640 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 861.445969] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 861.446431] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg f79f33742c6e4499996563211d3a9b04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.457408] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e2a1c5a7-34a1-4520-9b37-70f6087f0314 could not be found. [ 861.457605] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 861.457775] env[62736]: INFO nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Took 0.03 seconds to destroy the instance on the hypervisor. [ 861.458002] env[62736]: DEBUG oslo.service.loopingcall [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 861.458477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f79f33742c6e4499996563211d3a9b04 [ 861.458801] env[62736]: DEBUG nova.compute.manager [-] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 861.458893] env[62736]: DEBUG nova.network.neutron [-] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 861.482995] env[62736]: DEBUG nova.network.neutron [-] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.621201] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 6a2c1c84050948578c4c7c50ecc8c1c9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.656115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a2c1c84050948578c4c7c50ecc8c1c9 [ 861.728670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a75e0133558b4fc692fc9141639b392b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.736106] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a75e0133558b4fc692fc9141639b392b [ 861.849023] env[62736]: DEBUG oslo_concurrency.lockutils [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.013s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 861.849426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-346a9f43-e251-431e-ab08-cde60afc8b34 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 28f427cbf3fc41ada4a6853290e4d94d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.850349] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.270s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 861.852271] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg dc5e580ceb90404da9eb185c21a2d1e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.865108] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28f427cbf3fc41ada4a6853290e4d94d [ 861.894420] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc5e580ceb90404da9eb185c21a2d1e0 [ 861.950815] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-d866dcf0-c08b-4b12-be83-8de3e69c439f" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 861.950815] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 861.950815] env[62736]: DEBUG nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 861.950815] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 861.969703] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.970244] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a8cd199c9e2a41f49ddb5e6624cde23d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 861.976167] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8cd199c9e2a41f49ddb5e6624cde23d [ 862.147121] env[62736]: INFO nova.scheduler.client.report [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Deleted allocations for instance 76c79f9e-7ce6-4af9-b937-8e4bd32d39fe [ 862.152971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Expecting reply to msg 2b2f1f6b048b4cc8827552a3486c7014 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 862.162178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b2f1f6b048b4cc8827552a3486c7014 [ 862.229975] env[62736]: DEBUG nova.network.neutron [-] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.230618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 114a30262519468ca1d2e1cc4ab3a706 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 862.237540] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 114a30262519468ca1d2e1cc4ab3a706 [ 862.472795] env[62736]: DEBUG nova.network.neutron [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.473286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 02fe995175ca41188086645139bf14b5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 862.485490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02fe995175ca41188086645139bf14b5 [ 862.549916] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1bb30e41-a4e5-42a8-bc88-6d35c1615d52 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.557530] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9fda043e-65ec-4d0b-959d-77671f537199 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.586941] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5cf6575-c7ba-4f8c-aeec-b00e2c033367 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.593862] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e7c25be-06ad-4e90-b24e-64aa92bef1d6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 862.607458] env[62736]: DEBUG nova.compute.provider_tree [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 862.607945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg c1b7baf2911247fbaf92423f3afbbb97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 862.614730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1b7baf2911247fbaf92423f3afbbb97 [ 862.655193] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f5be47be-422f-4152-8cb3-4890719d2ec6 tempest-ImagesOneServerNegativeTestJSON-1164545313 tempest-ImagesOneServerNegativeTestJSON-1164545313-project-member] Lock "76c79f9e-7ce6-4af9-b937-8e4bd32d39fe" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 127.704s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 862.655873] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 3a3891bfa907454c94848d9c71679674 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 862.665045] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a3891bfa907454c94848d9c71679674 [ 862.733163] env[62736]: INFO nova.compute.manager [-] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Took 1.27 seconds to deallocate network for instance. [ 862.735905] env[62736]: DEBUG nova.compute.claims [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 862.736141] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 862.978026] env[62736]: INFO nova.compute.manager [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: d866dcf0-c08b-4b12-be83-8de3e69c439f] Took 1.03 seconds to deallocate network for instance. [ 862.979958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 60b29a5d29f645af9adfb86e0522375d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.017510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60b29a5d29f645af9adfb86e0522375d [ 863.110995] env[62736]: DEBUG nova.scheduler.client.report [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 863.113975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a8b5c38b833945ed9190e4d6aac8b2fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.125022] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8b5c38b833945ed9190e4d6aac8b2fe [ 863.158017] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 863.160221] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 65fd8a572ec64b6c927f931bd57cfabf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.194326] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65fd8a572ec64b6c927f931bd57cfabf [ 863.485685] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2d20a87c2ffa4054bf94d512e37173e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.519365] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d20a87c2ffa4054bf94d512e37173e3 [ 863.617033] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.766s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 863.617453] env[62736]: ERROR nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Traceback (most recent call last): [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.driver.spawn(context, instance, image_meta, [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] vm_ref = self.build_virtual_machine(instance, [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] vif_infos = vmwarevif.get_vif_info(self._session, [ 863.617453] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] for vif in network_info: [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return self._sync_wrapper(fn, *args, **kwargs) [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.wait() [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self[:] = self._gt.wait() [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return self._exit_event.wait() [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] current.throw(*self._exc) [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 863.617813] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] result = function(*args, **kwargs) [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] return func(*args, **kwargs) [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise e [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] nwinfo = self.network_api.allocate_for_instance( [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] created_port_ids = self._update_ports_for_instance( [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] with excutils.save_and_reraise_exception(): [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] self.force_reraise() [ 863.618194] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise self.value [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] updated_port = self._update_port( [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] _ensure_no_port_binding_failure(port) [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] raise exception.PortBindingFailed(port_id=port['id']) [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] nova.exception.PortBindingFailed: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. [ 863.618561] env[62736]: ERROR nova.compute.manager [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] [ 863.618561] env[62736]: DEBUG nova.compute.utils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 863.620604] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 14.952s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.620945] env[62736]: DEBUG nova.objects.instance [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lazy-loading 'resources' on Instance uuid 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 863.621662] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg e7563886a3d749f2857c44b908ce2606 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.623305] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Build of instance ddcf62db-af40-49d2-9465-66a10f0f2e2d was re-scheduled: Binding failed for port 205ec1e4-aa48-4292-9a63-d00ef691588a, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 863.623806] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 863.624123] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 863.624414] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 863.624956] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 863.625636] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 1d277a50f0e244a69f9675633c399c3e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 863.629438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e7563886a3d749f2857c44b908ce2606 [ 863.634768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d277a50f0e244a69f9675633c399c3e [ 863.682313] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.009651] env[62736]: INFO nova.scheduler.client.report [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance d866dcf0-c08b-4b12-be83-8de3e69c439f [ 864.015778] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a5193c66b5c34ba99a241b5303a8d7ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.040165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5193c66b5c34ba99a241b5303a8d7ea [ 864.151928] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.336473] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0568958-f87d-4040-9baf-062c4ef74d73 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.344596] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a034d133-a071-46a7-8a70-d7deba5b15d7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.383874] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fc19938-4454-4d9b-bd75-41073d00670c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.403091] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed6919a7-b83d-4525-a799-50b3eac6b34f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 864.416558] env[62736]: DEBUG nova.compute.provider_tree [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 864.417069] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg cf023bc9fcdd45d8b620b98da0df8047 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.428126] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.428748] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 6c2991f5dedd417398927262ccbd2d81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.432335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf023bc9fcdd45d8b620b98da0df8047 [ 864.437300] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c2991f5dedd417398927262ccbd2d81 [ 864.517810] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c8ab6928-28f5-4864-a1c5-09783c0058da tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "d866dcf0-c08b-4b12-be83-8de3e69c439f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 128.715s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.518521] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 2abc3d297505446cad21a0ada9e078f5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.528718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2abc3d297505446cad21a0ada9e078f5 [ 864.919535] env[62736]: DEBUG nova.scheduler.client.report [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 864.921987] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 85a6fa8561164113a68157b25956353d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.931061] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-ddcf62db-af40-49d2-9465-66a10f0f2e2d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 864.931270] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 864.931446] env[62736]: DEBUG nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 864.931605] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 864.936692] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85a6fa8561164113a68157b25956353d [ 864.953839] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.955500] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 56fcec6d7b0f4270b984211120280b7d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 864.962228] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 56fcec6d7b0f4270b984211120280b7d [ 865.021163] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 865.023032] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg a519d8fbe6014224a63fb89ddefec71b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.057029] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a519d8fbe6014224a63fb89ddefec71b [ 865.431835] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.805s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 865.431835] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.563s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.431835] env[62736]: INFO nova.compute.claims [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 865.431835] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 8098d0cd832c419db2f34f1ae1d39163 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.442080] env[62736]: INFO nova.scheduler.client.report [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Deleted allocations for instance 5d2465e7-e473-46e9-a8ce-cc58dcdbd28f [ 865.446208] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg b373dc15096b4ff6a054e61e632c99ac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.456792] env[62736]: DEBUG nova.network.neutron [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 865.457383] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg fd845c8d505a44ea834a4e2d9012159f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.466046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8098d0cd832c419db2f34f1ae1d39163 [ 865.468606] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd845c8d505a44ea834a4e2d9012159f [ 865.493387] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b373dc15096b4ff6a054e61e632c99ac [ 865.538696] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.660829] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "c1001bab-08b9-4bb9-b737-ba7fa5f929f0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.660829] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "c1001bab-08b9-4bb9-b737-ba7fa5f929f0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.739154] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.739651] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.934756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 2d5919cc444746e9ba49a7c657d5338c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.943059] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d5919cc444746e9ba49a7c657d5338c [ 865.957075] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Lock "5d2465e7-e473-46e9-a8ce-cc58dcdbd28f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 20.975s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 865.957665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9ae48bd4-015b-467c-89bb-007381912e36 tempest-ServerShowV257Test-1223842529 tempest-ServerShowV257Test-1223842529-project-member] Expecting reply to msg 6282e1639df340438106af5ff82b3122 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.960705] env[62736]: INFO nova.compute.manager [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: ddcf62db-af40-49d2-9465-66a10f0f2e2d] Took 1.03 seconds to deallocate network for instance. [ 865.962402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 60f93ef62e7646f98f68a64d206c492b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 865.968339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6282e1639df340438106af5ff82b3122 [ 865.994260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60f93ef62e7646f98f68a64d206c492b [ 866.467204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 8d9c14a659f342a2989ff1d8dfa6e323 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 866.512650] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d9c14a659f342a2989ff1d8dfa6e323 [ 866.665400] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30d3038a-d13d-4a2d-af39-b4479ac39c85 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.670802] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93adf567-f5d5-480d-aa7c-6bfe44174b16 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.703315] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7ff2653-7ee1-42e5-9744-ef69546c4489 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.710760] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1162055c-45a8-4c4b-97a3-8719d514f771 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 866.723972] env[62736]: DEBUG nova.compute.provider_tree [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 866.724692] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 68dcf2325a0a4c3fac202358f7fef6d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 866.731454] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68dcf2325a0a4c3fac202358f7fef6d7 [ 866.994672] env[62736]: INFO nova.scheduler.client.report [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Deleted allocations for instance ddcf62db-af40-49d2-9465-66a10f0f2e2d [ 867.001287] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 01be5a58f8764e418e77b3fea8b8ad7c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 867.013381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01be5a58f8764e418e77b3fea8b8ad7c [ 867.227173] env[62736]: DEBUG nova.scheduler.client.report [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 867.229687] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg d91e672921fb42eebb6f7328a42b0778 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 867.242086] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d91e672921fb42eebb6f7328a42b0778 [ 867.502957] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2b284dc3-18b8-4c53-9263-484078d72793 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "ddcf62db-af40-49d2-9465-66a10f0f2e2d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 121.079s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.503548] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 242db585b1284ce6b5b089fac63c8564 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 867.513099] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 242db585b1284ce6b5b089fac63c8564 [ 867.732934] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.305s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.735729] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 867.735729] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 77dd12db0c754f978ee1a143aed48c9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 867.736237] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 18.687s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 867.736901] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.737123] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 867.737464] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 15.916s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 867.739266] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 81d7c47e253b47b6adf2f244144e7117 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 867.741106] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57ab1796-c5d6-4ad2-a6b4-466312b5a573 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.750852] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f4d69792-c7ed-4b2d-b5c9-cb91f7786bfb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.764874] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f3a6209-2925-4f9a-b1ac-113ee012709c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.770976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77dd12db0c754f978ee1a143aed48c9a [ 867.771536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81d7c47e253b47b6adf2f244144e7117 [ 867.772580] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-862ad043-9587-431c-8fb3-8a70e36221f1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 867.803928] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181758MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 867.804127] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 868.005700] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 868.007398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg c0b467148fe14d68a44de018e886d7f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 868.040394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0b467148fe14d68a44de018e886d7f3 [ 868.241905] env[62736]: DEBUG nova.compute.utils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 868.242579] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 89e4f16162114567808a0d44fad6c199 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 868.243546] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 868.243709] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 868.256890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89e4f16162114567808a0d44fad6c199 [ 868.323235] env[62736]: DEBUG nova.policy [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'd68b7999531b4acb88e76d4393cee695', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '17dd5c4ec5e54de8867ffaa08754b7cd', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 868.430438] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1581160-9293-4268-9e50-731ab39f1b56 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.437809] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b360028-a1d5-4eb9-9c40-bddfa6356944 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.470986] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00e3d3eb-1efc-482a-bf30-605435c6d264 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.477982] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6cae66f-9a25-4288-9ae6-6e6bedb4b439 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 868.492096] env[62736]: DEBUG nova.compute.provider_tree [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 868.492709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg fb6f94da1f32407f9c217e45a28164f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 868.500290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb6f94da1f32407f9c217e45a28164f2 [ 868.524246] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 868.670269] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Successfully created port: 1512881e-a7b1-4caf-adc3-f4d2e248d3cc {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 868.749475] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 868.749475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 39426c60c6a54d9797cbe9cf4831d15e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 868.786622] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 39426c60c6a54d9797cbe9cf4831d15e [ 868.848761] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 868.849240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 868.995846] env[62736]: DEBUG nova.scheduler.client.report [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 868.998611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 6952a7bbec1b41938fae411e6cf6e71b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.011666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6952a7bbec1b41938fae411e6cf6e71b [ 869.254150] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 132b407b54f941b793995a8528a88eb0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.290489] env[62736]: DEBUG nova.compute.manager [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Received event network-changed-1512881e-a7b1-4caf-adc3-f4d2e248d3cc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 869.291197] env[62736]: DEBUG nova.compute.manager [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Refreshing instance network info cache due to event network-changed-1512881e-a7b1-4caf-adc3-f4d2e248d3cc. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 869.291570] env[62736]: DEBUG oslo_concurrency.lockutils [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] Acquiring lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.291881] env[62736]: DEBUG oslo_concurrency.lockutils [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] Acquired lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.292209] env[62736]: DEBUG nova.network.neutron [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Refreshing network info cache for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 869.292876] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] Expecting reply to msg 862774e97a85434ca7d471a24cf21f48 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.300679] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 862774e97a85434ca7d471a24cf21f48 [ 869.310179] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 132b407b54f941b793995a8528a88eb0 [ 869.501232] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.764s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 869.501900] env[62736]: ERROR nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Traceback (most recent call last): [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.driver.spawn(context, instance, image_meta, [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] vm_ref = self.build_virtual_machine(instance, [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] vif_infos = vmwarevif.get_vif_info(self._session, [ 869.501900] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] for vif in network_info: [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self._sync_wrapper(fn, *args, **kwargs) [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.wait() [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self[:] = self._gt.wait() [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self._exit_event.wait() [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] result = hub.switch() [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 869.502206] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return self.greenlet.switch() [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] result = function(*args, **kwargs) [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] return func(*args, **kwargs) [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise e [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] nwinfo = self.network_api.allocate_for_instance( [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] created_port_ids = self._update_ports_for_instance( [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] with excutils.save_and_reraise_exception(): [ 869.502517] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] self.force_reraise() [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise self.value [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] updated_port = self._update_port( [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] _ensure_no_port_binding_failure(port) [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] raise exception.PortBindingFailed(port_id=port['id']) [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] nova.exception.PortBindingFailed: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. [ 869.502829] env[62736]: ERROR nova.compute.manager [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] [ 869.503101] env[62736]: DEBUG nova.compute.utils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 869.503800] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.385s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 869.505521] env[62736]: INFO nova.compute.claims [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 869.507008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 7a4ddda3f0eb4eb2bbc0bc6b638b2050 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.508302] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Build of instance 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb was re-scheduled: Binding failed for port 08ad7f7a-3cb5-49b9-a534-2b18dbad7bc8, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 869.508872] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 869.508995] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.509140] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquired lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 869.509294] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 869.509684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 33e6622b79f94c18b5b4f41e47d1d563 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.515927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33e6622b79f94c18b5b4f41e47d1d563 [ 869.541778] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a4ddda3f0eb4eb2bbc0bc6b638b2050 [ 869.575362] env[62736]: ERROR nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 869.575362] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.575362] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.575362] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.575362] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.575362] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.575362] env[62736]: ERROR nova.compute.manager raise self.value [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.575362] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 869.575362] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.575362] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 869.575794] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.575794] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 869.575794] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 869.575794] env[62736]: ERROR nova.compute.manager [ 869.575794] env[62736]: Traceback (most recent call last): [ 869.575794] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 869.575794] env[62736]: listener.cb(fileno) [ 869.575794] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.575794] env[62736]: result = function(*args, **kwargs) [ 869.575794] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.575794] env[62736]: return func(*args, **kwargs) [ 869.575794] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.575794] env[62736]: raise e [ 869.575794] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.575794] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 869.575794] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.575794] env[62736]: created_port_ids = self._update_ports_for_instance( [ 869.575794] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.575794] env[62736]: with excutils.save_and_reraise_exception(): [ 869.575794] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.575794] env[62736]: self.force_reraise() [ 869.575794] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.575794] env[62736]: raise self.value [ 869.575794] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.575794] env[62736]: updated_port = self._update_port( [ 869.575794] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.575794] env[62736]: _ensure_no_port_binding_failure(port) [ 869.575794] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.575794] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 869.576516] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 869.576516] env[62736]: Removing descriptor: 17 [ 869.758225] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 869.782835] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 869.783084] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 869.783232] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 869.783406] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 869.783549] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 869.783687] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 869.783890] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 869.784065] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 869.784240] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 869.784399] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 869.784569] env[62736]: DEBUG nova.virt.hardware [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 869.785403] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00b01cf0-74dd-4af7-9667-15dd290b3825 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.794887] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8686de94-c8b5-4b13-88bb-6e4cac033ebf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 869.816836] env[62736]: ERROR nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Traceback (most recent call last): [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] yield resources [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.driver.spawn(context, instance, image_meta, [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] vm_ref = self.build_virtual_machine(instance, [ 869.816836] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] vif_infos = vmwarevif.get_vif_info(self._session, [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] for vif in network_info: [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return self._sync_wrapper(fn, *args, **kwargs) [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.wait() [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self[:] = self._gt.wait() [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return self._exit_event.wait() [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 869.817291] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] current.throw(*self._exc) [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] result = function(*args, **kwargs) [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return func(*args, **kwargs) [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise e [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] nwinfo = self.network_api.allocate_for_instance( [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] created_port_ids = self._update_ports_for_instance( [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] with excutils.save_and_reraise_exception(): [ 869.817732] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.force_reraise() [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise self.value [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] updated_port = self._update_port( [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] _ensure_no_port_binding_failure(port) [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise exception.PortBindingFailed(port_id=port['id']) [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 869.818195] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] [ 869.818195] env[62736]: INFO nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Terminating instance [ 869.820066] env[62736]: DEBUG nova.network.neutron [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 869.822691] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquiring lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 869.906507] env[62736]: DEBUG nova.network.neutron [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 869.907034] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] Expecting reply to msg 684bddfb433f49728ff8f0f22775ecf8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 869.916019] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 684bddfb433f49728ff8f0f22775ecf8 [ 870.012994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 4e89797867854b7881b5143884b34fdc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 870.021924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e89797867854b7881b5143884b34fdc [ 870.027933] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.106372] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.106917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 1f90dd78efbb4b46a453c1cb70c31de1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 870.119216] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f90dd78efbb4b46a453c1cb70c31de1 [ 870.409995] env[62736]: DEBUG oslo_concurrency.lockutils [req-76c5148a-b301-46fb-947b-292361f3fdcf req-5750e7bb-223c-447f-90e6-f50352a05ec9 service nova] Releasing lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.410440] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquired lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 870.410625] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 870.411051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg e01012bd2abe45d2a09ab0f1086f6c01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 870.417482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e01012bd2abe45d2a09ab0f1086f6c01 [ 870.608962] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Releasing lock "refresh_cache-9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 870.609197] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 870.609376] env[62736]: DEBUG nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 870.609598] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 870.623792] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 870.624366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 8719356534664cfaba2888f8c6566098 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 870.632139] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8719356534664cfaba2888f8c6566098 [ 870.714507] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdd95973-e61d-4b49-866a-c3f1fd700d46 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.722310] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d22a8690-6579-4a7b-a62f-3c6ea174370a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.751286] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59cb009f-47d5-4967-95cd-bc210283347a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.758225] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96f51d5c-b2a4-410a-926a-814701eac46c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 870.772150] env[62736]: DEBUG nova.compute.provider_tree [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 870.772643] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 5a0ab9654f3e4e778fd019df8c56df41 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 870.783596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a0ab9654f3e4e778fd019df8c56df41 [ 870.928299] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 871.008495] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.009040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg dc1ead06c34f444bb59670d91779d565 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.018917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc1ead06c34f444bb59670d91779d565 [ 871.127511] env[62736]: DEBUG nova.network.neutron [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.128204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 963638c74f1647b7a4a909db77a394d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.136317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 963638c74f1647b7a4a909db77a394d2 [ 871.275790] env[62736]: DEBUG nova.scheduler.client.report [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 871.278508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 1f6a5195ea364dd0b77e203d4512fb1f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.289894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f6a5195ea364dd0b77e203d4512fb1f [ 871.314769] env[62736]: DEBUG nova.compute.manager [req-ab69f30b-bb89-4bb1-a260-894a20e05eac req-53bd2fe1-cae7-4db3-8a87-9fbe4b4d272f service nova] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Received event network-vif-deleted-1512881e-a7b1-4caf-adc3-f4d2e248d3cc {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 871.511189] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Releasing lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 871.511649] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 871.511848] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 871.512182] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-800fdc13-a22d-4000-b7d6-a63488e51741 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 871.521458] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dcf2ed46-1258-410e-bc69-2a6f22492e30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 871.541915] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7d7cc825-7dcc-4b21-bf4c-886de451863a could not be found. [ 871.542167] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 871.542353] env[62736]: INFO nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Took 0.03 seconds to destroy the instance on the hypervisor. [ 871.542589] env[62736]: DEBUG oslo.service.loopingcall [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 871.543724] env[62736]: DEBUG nova.compute.manager [-] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 871.543810] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 871.559447] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 871.559922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9a9d8b4c4ab848bab5589258fae1ccf9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.566416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a9d8b4c4ab848bab5589258fae1ccf9 [ 871.636235] env[62736]: INFO nova.compute.manager [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb] Took 1.02 seconds to deallocate network for instance. [ 871.636235] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 7522122da6884584800c42b54f8e5199 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.671682] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7522122da6884584800c42b54f8e5199 [ 871.789976] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.283s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.789976] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 871.789976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg caa54a3141f04baf91df994b288fdb37 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.789976] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.015s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.791905] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 7ca80f8e679c4abdb45d64fc173a8f14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 871.825590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ca80f8e679c4abdb45d64fc173a8f14 [ 871.844013] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg caa54a3141f04baf91df994b288fdb37 [ 872.063603] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 872.063603] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 760459ed946349f59204720eccdaf98d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.072213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 760459ed946349f59204720eccdaf98d [ 872.142219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 66e149855e774f1cb9c1d659b4ffffd5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.171803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66e149855e774f1cb9c1d659b4ffffd5 [ 872.298753] env[62736]: DEBUG nova.compute.utils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 872.298753] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 38211d05cc4c4bcc8be503955039cc1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.302336] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 872.302694] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 872.308360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 38211d05cc4c4bcc8be503955039cc1e [ 872.358600] env[62736]: DEBUG nova.policy [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '52362e53439340569e09b18293dd5a3a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c41cd9d2bbc4443db64facd9d3f956d1', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 872.492114] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5013e88f-70d6-4391-ad78-e444f07db835 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.500714] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49c1b4e1-90f6-4e2b-86e4-e62b045d6602 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.537845] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f557182-a4c9-42f1-bdeb-4ecd5f00c0c9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.545969] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed1df325-4e53-4c7c-adbd-39c82027340a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 872.561297] env[62736]: DEBUG nova.compute.provider_tree [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 872.562138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d4e078c65d6e48d2909874be72291214 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.565997] env[62736]: INFO nova.compute.manager [-] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Took 1.02 seconds to deallocate network for instance. [ 872.568016] env[62736]: DEBUG nova.compute.claims [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 872.568338] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 872.569510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d4e078c65d6e48d2909874be72291214 [ 872.680983] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Successfully created port: 7f9f039c-f6fc-4bc1-a2fa-317383f60164 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 872.683784] env[62736]: INFO nova.scheduler.client.report [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Deleted allocations for instance 9f51bbbd-a116-4ebb-b286-4f7db9dbeedb [ 872.694174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg a8af0d0c941947b2800ec0d1c0557b5e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.710376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8af0d0c941947b2800ec0d1c0557b5e [ 872.803397] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 872.805169] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 775c7f3785874bf3b2bee83c1be4f3a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 872.847217] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 775c7f3785874bf3b2bee83c1be4f3a1 [ 873.068939] env[62736]: DEBUG nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 873.068939] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg b1a89a782a8d4390a7d487a0f38c666c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.081051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1a89a782a8d4390a7d487a0f38c666c [ 873.197762] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6d093002-e90d-432e-b9f5-2bf65da037d1 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "9f51bbbd-a116-4ebb-b286-4f7db9dbeedb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 126.239s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.197762] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 3c21b3af71764394a43ce3269909253b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.205411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c21b3af71764394a43ce3269909253b [ 873.313149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg ebc8c59e5fca4a0dae2b8bdb106f70dd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.350184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ebc8c59e5fca4a0dae2b8bdb106f70dd [ 873.503557] env[62736]: DEBUG nova.compute.manager [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Received event network-changed-7f9f039c-f6fc-4bc1-a2fa-317383f60164 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 873.503557] env[62736]: DEBUG nova.compute.manager [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Refreshing instance network info cache due to event network-changed-7f9f039c-f6fc-4bc1-a2fa-317383f60164. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 873.503557] env[62736]: DEBUG oslo_concurrency.lockutils [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] Acquiring lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.503557] env[62736]: DEBUG oslo_concurrency.lockutils [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] Acquired lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.503557] env[62736]: DEBUG nova.network.neutron [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Refreshing network info cache for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 873.503910] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] Expecting reply to msg 291e164a0ad04552a3e7940fc3fd4e1c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.510525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 291e164a0ad04552a3e7940fc3fd4e1c [ 873.570454] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.781s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 873.574877] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Traceback (most recent call last): [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.driver.spawn(context, instance, image_meta, [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] vm_ref = self.build_virtual_machine(instance, [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.574877] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] for vif in network_info: [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return self._sync_wrapper(fn, *args, **kwargs) [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.wait() [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self[:] = self._gt.wait() [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return self._exit_event.wait() [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] current.throw(*self._exc) [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.575240] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] result = function(*args, **kwargs) [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] return func(*args, **kwargs) [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise e [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] nwinfo = self.network_api.allocate_for_instance( [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] created_port_ids = self._update_ports_for_instance( [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] with excutils.save_and_reraise_exception(): [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] self.force_reraise() [ 873.575675] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise self.value [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] updated_port = self._update_port( [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] _ensure_no_port_binding_failure(port) [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] raise exception.PortBindingFailed(port_id=port['id']) [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] nova.exception.PortBindingFailed: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. [ 873.576070] env[62736]: ERROR nova.compute.manager [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] [ 873.576070] env[62736]: DEBUG nova.compute.utils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 873.577534] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 16.295s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.578611] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 545a11ad46a14e4d9a34303cb4e317cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.579989] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Build of instance 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e was re-scheduled: Binding failed for port 6d2239ac-5dcb-4286-baf3-2cb4fd85180d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 873.580423] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 873.580640] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 873.580776] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 873.580924] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 873.581273] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2701c67ee2c34f1ba56ef366804a93f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.586912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2701c67ee2c34f1ba56ef366804a93f4 [ 873.610853] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 545a11ad46a14e4d9a34303cb4e317cf [ 873.699596] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 873.701538] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 8d85820fd3ec41d3a566bc91bb4ebcba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 873.705538] env[62736]: ERROR nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 873.705538] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.705538] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.705538] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.705538] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.705538] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.705538] env[62736]: ERROR nova.compute.manager raise self.value [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.705538] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 873.705538] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.705538] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 873.706032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.706032] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 873.706032] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 873.706032] env[62736]: ERROR nova.compute.manager [ 873.706032] env[62736]: Traceback (most recent call last): [ 873.706032] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 873.706032] env[62736]: listener.cb(fileno) [ 873.706032] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.706032] env[62736]: result = function(*args, **kwargs) [ 873.706032] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.706032] env[62736]: return func(*args, **kwargs) [ 873.706032] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.706032] env[62736]: raise e [ 873.706032] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.706032] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 873.706032] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.706032] env[62736]: created_port_ids = self._update_ports_for_instance( [ 873.706032] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.706032] env[62736]: with excutils.save_and_reraise_exception(): [ 873.706032] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.706032] env[62736]: self.force_reraise() [ 873.706032] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.706032] env[62736]: raise self.value [ 873.706032] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.706032] env[62736]: updated_port = self._update_port( [ 873.706032] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.706032] env[62736]: _ensure_no_port_binding_failure(port) [ 873.706032] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.706032] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 873.706798] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 873.706798] env[62736]: Removing descriptor: 17 [ 873.733567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d85820fd3ec41d3a566bc91bb4ebcba [ 873.815678] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 873.841057] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 873.841297] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 873.841567] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 873.841774] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 873.841937] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 873.842083] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 873.842282] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 873.842437] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 873.842644] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 873.842811] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 873.842981] env[62736]: DEBUG nova.virt.hardware [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 873.843851] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8512fcb8-f249-4d50-b937-616e8df9d549 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.852488] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1febba9-35df-44d3-ace1-cbbe4a7f2381 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 873.866735] env[62736]: ERROR nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Traceback (most recent call last): [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] yield resources [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.driver.spawn(context, instance, image_meta, [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] vm_ref = self.build_virtual_machine(instance, [ 873.866735] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] vif_infos = vmwarevif.get_vif_info(self._session, [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] for vif in network_info: [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return self._sync_wrapper(fn, *args, **kwargs) [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.wait() [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self[:] = self._gt.wait() [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return self._exit_event.wait() [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 873.867159] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] current.throw(*self._exc) [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] result = function(*args, **kwargs) [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return func(*args, **kwargs) [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise e [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] nwinfo = self.network_api.allocate_for_instance( [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] created_port_ids = self._update_ports_for_instance( [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] with excutils.save_and_reraise_exception(): [ 873.867604] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.force_reraise() [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise self.value [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] updated_port = self._update_port( [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] _ensure_no_port_binding_failure(port) [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise exception.PortBindingFailed(port_id=port['id']) [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 873.867954] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] [ 873.867954] env[62736]: INFO nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Terminating instance [ 873.869030] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquiring lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 874.018403] env[62736]: DEBUG nova.network.neutron [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.110268] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.122450] env[62736]: DEBUG nova.network.neutron [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.122961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] Expecting reply to msg a6cd066ae35c4b509489af4ab0690cb8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.131113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6cd066ae35c4b509489af4ab0690cb8 [ 874.218179] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 874.270189] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85700809-4ed0-4a26-bcd6-cbd74d994b10 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.277930] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cba7617a-ef46-48a5-ba11-e1c4372291b1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.310778] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-485c7074-44b5-46f0-9ef8-ef964f90d9f9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.318123] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff8bc73b-5c7b-4fb9-959a-1733917b635e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 874.331068] env[62736]: DEBUG nova.compute.provider_tree [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 874.331571] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 63a1cc4605cd43bf8100ed87485f051f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.343822] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63a1cc4605cd43bf8100ed87485f051f [ 874.380999] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 874.381529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 155756ef93bd4f1c9b14deac462c404c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.389838] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 155756ef93bd4f1c9b14deac462c404c [ 874.625349] env[62736]: DEBUG oslo_concurrency.lockutils [req-e52f99ef-adea-410a-a0db-a802e7d7fb26 req-60889f77-e54d-4366-888b-1699884247be service nova] Releasing lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 874.625563] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquired lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 874.625784] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 874.626172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 4894fe914ccc4923984b32f8e36374d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.634536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4894fe914ccc4923984b32f8e36374d5 [ 874.834643] env[62736]: DEBUG nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 874.837360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d94f38c79b574cf39ebe4d673d685f84 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.848624] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d94f38c79b574cf39ebe4d673d685f84 [ 874.884593] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 874.884861] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 874.885072] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 874.885252] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 874.904066] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 874.904639] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg dddd696e66554a3a959b57f40ea41907 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 874.911550] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dddd696e66554a3a959b57f40ea41907 [ 875.145263] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.219122] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.219717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 92abfcac92ed43aca904ac11f99179b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.227805] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 92abfcac92ed43aca904ac11f99179b8 [ 875.339926] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.763s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 875.340656] env[62736]: ERROR nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Traceback (most recent call last): [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.driver.spawn(context, instance, image_meta, [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self._vmops.spawn(context, instance, image_meta, injected_files, [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] vm_ref = self.build_virtual_machine(instance, [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] vif_infos = vmwarevif.get_vif_info(self._session, [ 875.340656] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] for vif in network_info: [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return self._sync_wrapper(fn, *args, **kwargs) [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.wait() [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self[:] = self._gt.wait() [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return self._exit_event.wait() [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] current.throw(*self._exc) [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 875.341033] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] result = function(*args, **kwargs) [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] return func(*args, **kwargs) [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise e [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] nwinfo = self.network_api.allocate_for_instance( [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] created_port_ids = self._update_ports_for_instance( [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] with excutils.save_and_reraise_exception(): [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] self.force_reraise() [ 875.341405] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise self.value [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] updated_port = self._update_port( [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] _ensure_no_port_binding_failure(port) [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] raise exception.PortBindingFailed(port_id=port['id']) [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] nova.exception.PortBindingFailed: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. [ 875.341811] env[62736]: ERROR nova.compute.manager [instance: 1f90a84e-6c32-4846-8908-128323e33e98] [ 875.341811] env[62736]: DEBUG nova.compute.utils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 875.342611] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.793s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 875.344414] env[62736]: INFO nova.compute.claims [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 875.346634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 350fc9cd37bf494ea9fc1c4b45415c0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.347359] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Build of instance 1f90a84e-6c32-4846-8908-128323e33e98 was re-scheduled: Binding failed for port 8892036c-2583-407d-8dd1-b44a5d64495f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 875.347840] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 875.348116] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquiring lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 875.348298] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Acquired lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 875.348495] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 875.348892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 3319b4a5bd394c8e99baaa675959853b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.355373] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3319b4a5bd394c8e99baaa675959853b [ 875.380314] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 350fc9cd37bf494ea9fc1c4b45415c0a [ 875.406771] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.407281] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 1310031e0b5346d5ac5d919772ee7bb9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.415243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1310031e0b5346d5ac5d919772ee7bb9 [ 875.559258] env[62736]: DEBUG nova.compute.manager [req-26fda668-7eca-4148-910a-1e17f8062de8 req-c2a8c94f-b6d3-48c2-a8f3-16a07d8a9479 service nova] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Received event network-vif-deleted-7f9f039c-f6fc-4bc1-a2fa-317383f60164 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 875.722935] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Releasing lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 875.723387] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 875.723585] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 875.723901] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fb4f489e-277a-495f-b0f1-2046b1c78d97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.733079] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-324c9d5c-768a-4272-9d8d-41bdf166ada7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 875.754793] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 86be55c9-82d5-4a34-b628-4729c42d83c1 could not be found. [ 875.755138] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 875.755422] env[62736]: INFO nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Took 0.03 seconds to destroy the instance on the hypervisor. [ 875.755783] env[62736]: DEBUG oslo.service.loopingcall [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 875.756163] env[62736]: DEBUG nova.compute.manager [-] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 875.756363] env[62736]: DEBUG nova.network.neutron [-] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 875.770667] env[62736]: DEBUG nova.network.neutron [-] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.771209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 077e2461c8aa42a080a299c3208d6175 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.777469] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 077e2461c8aa42a080a299c3208d6175 [ 875.849733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg a076f39d41a443a88f0e717bd1c715c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.857052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a076f39d41a443a88f0e717bd1c715c6 [ 875.866486] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.909731] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e] Took 1.02 seconds to deallocate network for instance. [ 875.911746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 3f23fba27e86453bb2fce36f0a08b14a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.949405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f23fba27e86453bb2fce36f0a08b14a [ 875.952570] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.953046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 21659d4e34394521bfab3987d2c78b5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 875.961073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 21659d4e34394521bfab3987d2c78b5c [ 876.223557] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "6add8218-734e-4181-8ada-4657331024e5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 876.223834] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "6add8218-734e-4181-8ada-4657331024e5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 876.272768] env[62736]: DEBUG nova.network.neutron [-] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.273161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b4e784afcda640d6843ec7057269b3af in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 876.281168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4e784afcda640d6843ec7057269b3af [ 876.416190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 2bb9ce0ce294419b95c8533cffd23cfb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 876.457793] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Releasing lock "refresh_cache-1f90a84e-6c32-4846-8908-128323e33e98" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 876.457793] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 876.457793] env[62736]: DEBUG nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 876.457793] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 876.470947] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 876.471498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg d5bbaf25ea7540d2af2accdc61bbc594 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 876.472567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2bb9ce0ce294419b95c8533cffd23cfb [ 876.479949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5bbaf25ea7540d2af2accdc61bbc594 [ 876.528902] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd034466-ff78-4c12-a986-9795e0623356 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.536473] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd00dab0-e5ec-4f0c-8e7b-2c323b8f4ee0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.566897] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-459dcc9d-b189-4fbf-a120-1d752d29b97e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.573848] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e9d9f4-3d92-458e-9f7b-50c4ce226545 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 876.586414] env[62736]: DEBUG nova.compute.provider_tree [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 876.587077] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 26e3bdebce2b433682b119f9d385bfb7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 876.596921] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26e3bdebce2b433682b119f9d385bfb7 [ 876.775659] env[62736]: INFO nova.compute.manager [-] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Took 1.02 seconds to deallocate network for instance. [ 876.778038] env[62736]: DEBUG nova.compute.claims [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 876.778209] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 876.937775] env[62736]: INFO nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Deleted allocations for instance 1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e [ 876.944081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg f1214413b1f244c09a6053da8eed1772 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 876.961283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1214413b1f244c09a6053da8eed1772 [ 876.973847] env[62736]: DEBUG nova.network.neutron [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.974347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 451fd905dd214be49a7ca3b4d1e3d864 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.013340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 451fd905dd214be49a7ca3b4d1e3d864 [ 877.089083] env[62736]: DEBUG nova.scheduler.client.report [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 877.091563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg b66ad029d23348e9a62515f2357aa453 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.102338] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b66ad029d23348e9a62515f2357aa453 [ 877.453408] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "1f0cc9c9-0d9f-40ea-a2cf-ff94c793ff4e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 97.604s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 877.454044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 0f4a00a9cb6941628a9e66a561f43384 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.463049] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f4a00a9cb6941628a9e66a561f43384 [ 877.479686] env[62736]: INFO nova.compute.manager [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] [instance: 1f90a84e-6c32-4846-8908-128323e33e98] Took 1.02 seconds to deallocate network for instance. [ 877.481366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 26793ea4260144ea911100be93b387c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.512658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26793ea4260144ea911100be93b387c5 [ 877.594500] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.252s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 877.595045] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 877.596694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg fe95e3aeb7844716b411f51327b0e483 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.597882] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 14.862s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 877.599743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 084769d6d6064168ba7214667f600a25 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.628499] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe95e3aeb7844716b411f51327b0e483 [ 877.638241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 084769d6d6064168ba7214667f600a25 [ 877.958034] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 877.958519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 5c8d7a80e702469cbb1d9d0b9d574ebd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.986574] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg 93bace308cf9494ca85578a2ab908176 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 877.993379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c8d7a80e702469cbb1d9d0b9d574ebd [ 878.021385] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93bace308cf9494ca85578a2ab908176 [ 878.104943] env[62736]: DEBUG nova.compute.utils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 878.104943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 9b878173b725470184751c60713efd6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 878.108681] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 878.109459] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 878.113895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b878173b725470184751c60713efd6c [ 878.185537] env[62736]: DEBUG nova.policy [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'c43c33165931465494c0a7da171126e4', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'de11ac365f704c2986c782a5918f802f', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 878.354023] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5501dfe5-2090-4280-80f1-6e955092e5d2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.362567] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7457c57f-34ff-4aa8-912e-1f2a482dd81b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.401138] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a94de916-df62-4212-ac6d-253bdc321cc9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.409792] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6dcca694-0ff8-4903-884e-3193fbef3e57 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 878.423314] env[62736]: DEBUG nova.compute.provider_tree [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 878.423965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 463f216d66594955aa4263e1d747d9c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 878.431174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 463f216d66594955aa4263e1d747d9c8 [ 878.484296] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Successfully created port: 7b9d929d-0980-4cd6-ad6b-83398958648e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 878.486812] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 878.513264] env[62736]: INFO nova.scheduler.client.report [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Deleted allocations for instance 1f90a84e-6c32-4846-8908-128323e33e98 [ 878.519672] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Expecting reply to msg f40a213f73a8401289f185ffe23fa6aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 878.547112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f40a213f73a8401289f185ffe23fa6aa [ 878.609940] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 878.611747] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 52e4ce228acc42febcd427ebf849a79e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 878.656635] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52e4ce228acc42febcd427ebf849a79e [ 878.927219] env[62736]: DEBUG nova.scheduler.client.report [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 878.930028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 7a19bba0f41242a184bf871b6c586026 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 878.943085] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a19bba0f41242a184bf871b6c586026 [ 879.021837] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4a1e7b55-b8b9-4ba4-9d19-cfcdb42bb5c8 tempest-MultipleCreateTestJSON-781183485 tempest-MultipleCreateTestJSON-781183485-project-member] Lock "1f90a84e-6c32-4846-8908-128323e33e98" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 99.149s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.023180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg adb53117d39f48609f02afecb37ea64f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.035985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adb53117d39f48609f02afecb37ea64f [ 879.115937] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 962dc6a4bfcb4b919eb56a82b6daa212 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.133037] env[62736]: DEBUG nova.compute.manager [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Received event network-changed-7b9d929d-0980-4cd6-ad6b-83398958648e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 879.133289] env[62736]: DEBUG nova.compute.manager [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Refreshing instance network info cache due to event network-changed-7b9d929d-0980-4cd6-ad6b-83398958648e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 879.133548] env[62736]: DEBUG oslo_concurrency.lockutils [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] Acquiring lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 879.133735] env[62736]: DEBUG oslo_concurrency.lockutils [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] Acquired lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 879.133976] env[62736]: DEBUG nova.network.neutron [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Refreshing network info cache for port 7b9d929d-0980-4cd6-ad6b-83398958648e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 879.134491] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] Expecting reply to msg c577bbe1637b427e8b1f5230be6aeff9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.146726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c577bbe1637b427e8b1f5230be6aeff9 [ 879.155395] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 962dc6a4bfcb4b919eb56a82b6daa212 [ 879.311478] env[62736]: ERROR nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 879.311478] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 879.311478] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 879.311478] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 879.311478] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 879.311478] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 879.311478] env[62736]: ERROR nova.compute.manager raise self.value [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 879.311478] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 879.311478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 879.311478] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 879.311994] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 879.311994] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 879.311994] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 879.311994] env[62736]: ERROR nova.compute.manager [ 879.311994] env[62736]: Traceback (most recent call last): [ 879.311994] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 879.311994] env[62736]: listener.cb(fileno) [ 879.311994] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 879.311994] env[62736]: result = function(*args, **kwargs) [ 879.311994] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 879.311994] env[62736]: return func(*args, **kwargs) [ 879.311994] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 879.311994] env[62736]: raise e [ 879.311994] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 879.311994] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 879.311994] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 879.311994] env[62736]: created_port_ids = self._update_ports_for_instance( [ 879.311994] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 879.311994] env[62736]: with excutils.save_and_reraise_exception(): [ 879.311994] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 879.311994] env[62736]: self.force_reraise() [ 879.311994] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 879.311994] env[62736]: raise self.value [ 879.311994] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 879.311994] env[62736]: updated_port = self._update_port( [ 879.311994] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 879.311994] env[62736]: _ensure_no_port_binding_failure(port) [ 879.311994] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 879.311994] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 879.312758] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 879.312758] env[62736]: Removing descriptor: 16 [ 879.432513] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.834s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.433688] env[62736]: ERROR nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Traceback (most recent call last): [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.driver.spawn(context, instance, image_meta, [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self._vmops.spawn(context, instance, image_meta, injected_files, [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] vm_ref = self.build_virtual_machine(instance, [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] vif_infos = vmwarevif.get_vif_info(self._session, [ 879.433688] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] for vif in network_info: [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return self._sync_wrapper(fn, *args, **kwargs) [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.wait() [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self[:] = self._gt.wait() [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return self._exit_event.wait() [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] current.throw(*self._exc) [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 879.433998] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] result = function(*args, **kwargs) [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] return func(*args, **kwargs) [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise e [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] nwinfo = self.network_api.allocate_for_instance( [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] created_port_ids = self._update_ports_for_instance( [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] with excutils.save_and_reraise_exception(): [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] self.force_reraise() [ 879.434309] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise self.value [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] updated_port = self._update_port( [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] _ensure_no_port_binding_failure(port) [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] raise exception.PortBindingFailed(port_id=port['id']) [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] nova.exception.PortBindingFailed: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. [ 879.434678] env[62736]: ERROR nova.compute.manager [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] [ 879.434678] env[62736]: DEBUG nova.compute.utils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 879.435602] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 15.754s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.437068] env[62736]: INFO nova.compute.claims [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 879.439595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 4776980cdac343448226732b70b4e567 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.441028] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Build of instance e2a1c5a7-34a1-4520-9b37-70f6087f0314 was re-scheduled: Binding failed for port 1dd6ec9b-97da-4ae2-a996-2fe67cf26328, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 879.441520] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 879.441815] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquiring lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 879.446639] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Acquired lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 879.446639] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 879.446639] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 837d0f259de8433bb43ed253865ee8d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.448485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 837d0f259de8433bb43ed253865ee8d5 [ 879.481385] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4776980cdac343448226732b70b4e567 [ 879.524914] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 879.526712] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c33c6ff6afb147babc9089b2a6bb781f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.558209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c33c6ff6afb147babc9089b2a6bb781f [ 879.619007] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 879.646623] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 879.646881] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 879.647112] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 879.647357] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 879.647584] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 879.647793] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 879.648068] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 879.648291] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 879.648519] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 879.648743] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 879.648976] env[62736]: DEBUG nova.virt.hardware [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 879.649903] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9b78d43-7783-4e82-aafb-dd1a814ccd05 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.653530] env[62736]: DEBUG nova.network.neutron [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 879.661271] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d3b0a54-44fa-4b11-8780-67c2080a03a6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 879.675639] env[62736]: ERROR nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Traceback (most recent call last): [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] yield resources [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.driver.spawn(context, instance, image_meta, [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self._vmops.spawn(context, instance, image_meta, injected_files, [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] vm_ref = self.build_virtual_machine(instance, [ 879.675639] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] vif_infos = vmwarevif.get_vif_info(self._session, [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] for vif in network_info: [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return self._sync_wrapper(fn, *args, **kwargs) [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.wait() [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self[:] = self._gt.wait() [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return self._exit_event.wait() [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 879.676184] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] current.throw(*self._exc) [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] result = function(*args, **kwargs) [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return func(*args, **kwargs) [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise e [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] nwinfo = self.network_api.allocate_for_instance( [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] created_port_ids = self._update_ports_for_instance( [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] with excutils.save_and_reraise_exception(): [ 879.676564] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.force_reraise() [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise self.value [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] updated_port = self._update_port( [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] _ensure_no_port_binding_failure(port) [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise exception.PortBindingFailed(port_id=port['id']) [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 879.676969] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] [ 879.676969] env[62736]: INFO nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Terminating instance [ 879.678966] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 879.726242] env[62736]: DEBUG nova.network.neutron [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 879.726863] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] Expecting reply to msg feaf65cab57e41ccb6c8b60caf38337e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.734841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg feaf65cab57e41ccb6c8b60caf38337e [ 879.945964] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg cd667aab9e1b4bc6bb39a9ed2302feab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 879.959342] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd667aab9e1b4bc6bb39a9ed2302feab [ 879.969934] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 880.053849] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.125201] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 880.125680] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 9b99354fc1204087be277c4aaed9b50a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 880.135133] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b99354fc1204087be277c4aaed9b50a [ 880.229220] env[62736]: DEBUG oslo_concurrency.lockutils [req-14762ff9-d481-4d2a-aaba-3c43496634ec req-b69f70b2-9e7d-4cea-bb58-32a3a974639b service nova] Releasing lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 880.229820] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 880.230010] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 880.230581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 012dfcd04db643d9959ffff1c8aa0331 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 880.238673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 012dfcd04db643d9959ffff1c8aa0331 [ 880.635840] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Releasing lock "refresh_cache-e2a1c5a7-34a1-4520-9b37-70f6087f0314" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 880.636204] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 880.636271] env[62736]: DEBUG nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 880.636398] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 880.638566] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd53b178-33d2-4c86-bcc9-a4ad5953838a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.647045] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dae61b10-b40f-4d81-ab55-edb8e04f1d16 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.678802] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 880.679385] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg efbd9bb407104629966a8c0c9b11df64 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 880.683609] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66fd7c18-a6a7-4608-8291-9391f8d65923 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.686917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efbd9bb407104629966a8c0c9b11df64 [ 880.693112] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbc9e685-c33f-4f7b-ab0b-abce1ba0be2d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 880.708057] env[62736]: DEBUG nova.compute.provider_tree [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 880.708646] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg eb007f098189452ab13920069bdd508b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 880.718043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb007f098189452ab13920069bdd508b [ 880.767378] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 880.934926] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 880.935482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 5e85c3f3530447119ae7251af932724a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 880.947567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e85c3f3530447119ae7251af932724a [ 881.180259] env[62736]: DEBUG nova.compute.manager [req-6db8fdd4-0322-44a3-add5-f8164a280b95 req-38382f2f-ba23-4736-8d35-443e1561ed4a service nova] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Received event network-vif-deleted-7b9d929d-0980-4cd6-ad6b-83398958648e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 881.189168] env[62736]: DEBUG nova.network.neutron [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 881.189168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 95ee125f6efb48cfbd900d2f1a1ba7a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.198095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95ee125f6efb48cfbd900d2f1a1ba7a8 [ 881.211631] env[62736]: DEBUG nova.scheduler.client.report [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 881.214319] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 0e7e3a87ccc04f61ab68e6f508ea7abf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.229126] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e7e3a87ccc04f61ab68e6f508ea7abf [ 881.441825] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 881.441825] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 881.441825] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 881.441825] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0aec5161-22e7-4266-8da7-40fb653388fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.449918] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-426253b5-c37f-476c-9337-30c42046edaa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 881.472159] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 53625c0d-203a-45f0-b899-5d2673413a53 could not be found. [ 881.472336] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 881.472529] env[62736]: INFO nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Took 0.03 seconds to destroy the instance on the hypervisor. [ 881.472790] env[62736]: DEBUG oslo.service.loopingcall [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 881.473014] env[62736]: DEBUG nova.compute.manager [-] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 881.473106] env[62736]: DEBUG nova.network.neutron [-] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 881.494194] env[62736]: DEBUG nova.network.neutron [-] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 881.494721] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg db8eec654a4d489c9ec11d917db48adb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.513291] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db8eec654a4d489c9ec11d917db48adb [ 881.690770] env[62736]: INFO nova.compute.manager [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] [instance: e2a1c5a7-34a1-4520-9b37-70f6087f0314] Took 1.05 seconds to deallocate network for instance. [ 881.692625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 679db1d17db848cfbfeed95a553e1ec7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.717844] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.282s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.718402] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 881.720228] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 26997cc4a1e4427b97232782212e2018 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.721725] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 16.183s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.723404] env[62736]: INFO nova.compute.claims [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 881.725827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 492763414f02440da6cae17110660271 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 881.744475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 679db1d17db848cfbfeed95a553e1ec7 [ 881.751728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26997cc4a1e4427b97232782212e2018 [ 881.765043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 492763414f02440da6cae17110660271 [ 882.011511] env[62736]: DEBUG nova.network.neutron [-] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 882.011511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a405f9be58c64e289c369b42909be01a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.019872] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a405f9be58c64e289c369b42909be01a [ 882.197655] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg 405c99b4f1614d5f8adc800f4a07d1cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.223082] env[62736]: DEBUG nova.compute.utils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 882.223707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg bacc1f8a95534435b748c52ab831b9cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.225236] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 882.225568] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 882.232058] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg d649e7f8a08744bca833bf359f2352ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.234248] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405c99b4f1614d5f8adc800f4a07d1cf [ 882.235122] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bacc1f8a95534435b748c52ab831b9cf [ 882.240476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d649e7f8a08744bca833bf359f2352ee [ 882.282243] env[62736]: DEBUG nova.policy [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '23c687bad35b409ab96bfd7cde54fd27', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9a349372836346d8983c95b50d980e37', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 882.513893] env[62736]: INFO nova.compute.manager [-] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Took 1.04 seconds to deallocate network for instance. [ 882.516250] env[62736]: DEBUG nova.compute.claims [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 882.516423] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.552750] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Successfully created port: f5acee40-6e6c-4fea-a969-034bb88c660e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 882.725575] env[62736]: INFO nova.scheduler.client.report [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Deleted allocations for instance e2a1c5a7-34a1-4520-9b37-70f6087f0314 [ 882.741465] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 882.743072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 4eb5bea87949422cb7cdb0315f14621f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.744324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Expecting reply to msg fbafa07602ee47599567929d1fee066d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.770608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fbafa07602ee47599567929d1fee066d [ 882.772736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4eb5bea87949422cb7cdb0315f14621f [ 882.934310] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7d3daca-2769-43f1-8389-97dbecc9764e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.942493] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9ade308-8eaa-49f8-bbe8-a078f7de2a1e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.970963] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64365020-800f-408e-9782-5017da2f8a90 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.977848] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d9cf4f0f-878b-4653-97eb-dc9e4a84b7eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 882.990624] env[62736]: DEBUG nova.compute.provider_tree [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 882.991224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg d044f5232f9041b9ba8dda26c144c0e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 882.997774] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d044f5232f9041b9ba8dda26c144c0e9 [ 883.248651] env[62736]: DEBUG oslo_concurrency.lockutils [None req-25b8c6eb-5bb3-4288-8541-76fd3b6fe1b4 tempest-ServersTestMultiNic-922746751 tempest-ServersTestMultiNic-922746751-project-member] Lock "e2a1c5a7-34a1-4520-9b37-70f6087f0314" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 87.403s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 883.251342] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 937412b8e9014117ad64c4840bb4c3b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.253390] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9fb12e74b9e64efc8977e03379193cf8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.262624] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9fb12e74b9e64efc8977e03379193cf8 [ 883.286356] env[62736]: DEBUG nova.compute.manager [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Received event network-changed-f5acee40-6e6c-4fea-a969-034bb88c660e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 883.286636] env[62736]: DEBUG nova.compute.manager [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Refreshing instance network info cache due to event network-changed-f5acee40-6e6c-4fea-a969-034bb88c660e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 883.286909] env[62736]: DEBUG oslo_concurrency.lockutils [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] Acquiring lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 883.287093] env[62736]: DEBUG oslo_concurrency.lockutils [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] Acquired lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 883.287288] env[62736]: DEBUG nova.network.neutron [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Refreshing network info cache for port f5acee40-6e6c-4fea-a969-034bb88c660e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 883.287772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] Expecting reply to msg a9151b4da5264abb9061564c0be2a760 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.289517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 937412b8e9014117ad64c4840bb4c3b3 [ 883.293868] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a9151b4da5264abb9061564c0be2a760 [ 883.370336] env[62736]: ERROR nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 883.370336] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 883.370336] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 883.370336] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 883.370336] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 883.370336] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 883.370336] env[62736]: ERROR nova.compute.manager raise self.value [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 883.370336] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 883.370336] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 883.370336] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 883.370896] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 883.370896] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 883.370896] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 883.370896] env[62736]: ERROR nova.compute.manager [ 883.370896] env[62736]: Traceback (most recent call last): [ 883.370896] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 883.370896] env[62736]: listener.cb(fileno) [ 883.370896] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 883.370896] env[62736]: result = function(*args, **kwargs) [ 883.370896] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 883.370896] env[62736]: return func(*args, **kwargs) [ 883.370896] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 883.370896] env[62736]: raise e [ 883.370896] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 883.370896] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 883.370896] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 883.370896] env[62736]: created_port_ids = self._update_ports_for_instance( [ 883.370896] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 883.370896] env[62736]: with excutils.save_and_reraise_exception(): [ 883.370896] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 883.370896] env[62736]: self.force_reraise() [ 883.370896] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 883.370896] env[62736]: raise self.value [ 883.370896] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 883.370896] env[62736]: updated_port = self._update_port( [ 883.370896] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 883.370896] env[62736]: _ensure_no_port_binding_failure(port) [ 883.370896] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 883.370896] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 883.371724] env[62736]: nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 883.371724] env[62736]: Removing descriptor: 16 [ 883.493569] env[62736]: DEBUG nova.scheduler.client.report [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 883.495906] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg a82c5c21f605467e81a54cf74648cd6e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.508505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a82c5c21f605467e81a54cf74648cd6e [ 883.754631] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 883.756966] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 883.758566] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg ced66ba4e00f416fa5939d76f0cc8d7e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.779164] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 883.779437] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 883.779585] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 883.779774] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 883.779916] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 883.780072] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 883.780276] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 883.780431] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 883.780593] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 883.780749] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 883.780918] env[62736]: DEBUG nova.virt.hardware [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 883.781769] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-170d9dc8-a277-4191-8b27-0bc062918680 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 883.789687] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbe7eb48-de9f-4afa-ba5a-b415b96a51a9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 883.793516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ced66ba4e00f416fa5939d76f0cc8d7e [ 883.807652] env[62736]: ERROR nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Traceback (most recent call last): [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] yield resources [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.driver.spawn(context, instance, image_meta, [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self._vmops.spawn(context, instance, image_meta, injected_files, [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] vm_ref = self.build_virtual_machine(instance, [ 883.807652] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] vif_infos = vmwarevif.get_vif_info(self._session, [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] for vif in network_info: [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return self._sync_wrapper(fn, *args, **kwargs) [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.wait() [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self[:] = self._gt.wait() [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return self._exit_event.wait() [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 883.808460] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] current.throw(*self._exc) [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] result = function(*args, **kwargs) [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return func(*args, **kwargs) [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise e [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] nwinfo = self.network_api.allocate_for_instance( [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] created_port_ids = self._update_ports_for_instance( [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] with excutils.save_and_reraise_exception(): [ 883.809300] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.force_reraise() [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise self.value [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] updated_port = self._update_port( [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] _ensure_no_port_binding_failure(port) [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise exception.PortBindingFailed(port_id=port['id']) [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 883.811887] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] [ 883.811887] env[62736]: INFO nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Terminating instance [ 883.812703] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 883.812703] env[62736]: DEBUG nova.network.neutron [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 883.897025] env[62736]: DEBUG nova.network.neutron [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 883.897607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] Expecting reply to msg 7c97dcb80c094a97b79c1f4c2c8c25fb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 883.906078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c97dcb80c094a97b79c1f4c2c8c25fb [ 883.998441] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.277s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 883.999002] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 884.000828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 5f8f8ae91289421c82bfca639fd935b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 884.001871] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 16.198s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 884.002620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg e195f767cc094881a038d7cfbe511c3d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 884.038433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e195f767cc094881a038d7cfbe511c3d [ 884.042227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f8f8ae91289421c82bfca639fd935b7 [ 884.279753] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 884.402293] env[62736]: DEBUG oslo_concurrency.lockutils [req-9b318670-ef30-4c3b-807a-c9e518610814 req-83189399-a08e-465c-945b-25906cee63ef service nova] Releasing lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 884.402293] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquired lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 884.402293] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 884.402293] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg e47efddb3b1c4538a2048f2d56367478 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 884.409677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e47efddb3b1c4538a2048f2d56367478 [ 884.512851] env[62736]: DEBUG nova.compute.utils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 884.512851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg c49550396964409599f0e9f0529dd0a8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 884.512851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 1ba312ce2cf04144a736de3c99b86478 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 884.512851] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 884.512851] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 884.517030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c49550396964409599f0e9f0529dd0a8 [ 884.521924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ba312ce2cf04144a736de3c99b86478 [ 884.553389] env[62736]: DEBUG nova.policy [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd65b3ab163145e1bf370aa0dc63f65c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8003d16b1cc048bc94407d028838a4f6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 884.853534] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Successfully created port: 57fb7c5e-9fe9-48df-ac1b-5a913e230491 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 884.920091] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 885.011684] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 885.013746] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 35cb8fcc7afa4fd5a1889504e230b74f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.025386] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 885.026011] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 89d895b32e0d4c23a5699e22b87c51e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.035548] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89d895b32e0d4c23a5699e22b87c51e0 [ 885.038523] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 7d7cc825-7dcc-4b21-bf4c-886de451863a actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 885.038676] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 86be55c9-82d5-4a34-b628-4729c42d83c1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 885.038794] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 53625c0d-203a-45f0-b899-5d2673413a53 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 885.038908] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 885.039017] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 570ee4e0-caeb-41f9-82d2-fa009f23581d actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 885.039613] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b3a62577da2a45fd98623a65df9c0d0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.053302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3a62577da2a45fd98623a65df9c0d0a [ 885.059795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35cb8fcc7afa4fd5a1889504e230b74f [ 885.316978] env[62736]: DEBUG nova.compute.manager [req-8266000d-487c-444b-89ab-2312ccad428e req-3951c9d2-c60f-4d5f-b6e9-5622ee59571b service nova] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Received event network-vif-deleted-f5acee40-6e6c-4fea-a969-034bb88c660e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 885.520799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 6d431206cbbe4aa688f1dcb34b51f500 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.531847] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Releasing lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 885.532244] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 885.532427] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 885.532705] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e7aba1d0-1ff8-40e8-923d-e056dc8fa0e0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.541646] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 7d047648-facb-4d58-acf7-a99a3f7adf65 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 885.542181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 62d527a42b0c417aaf261a340f46e2cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.545910] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bde62064-1d86-4266-b0be-5d49e0f0c63d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.566104] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62d527a42b0c417aaf261a340f46e2cf [ 885.567066] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance fa27cfe2-6f7f-4397-bcc5-147d6865f404 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 885.567619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 283535862c7a41aca9a4af367eecafc2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.570052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d431206cbbe4aa688f1dcb34b51f500 [ 885.579363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 283535862c7a41aca9a4af367eecafc2 [ 885.580326] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90 could not be found. [ 885.580326] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 885.580456] env[62736]: INFO nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Took 0.05 seconds to destroy the instance on the hypervisor. [ 885.580699] env[62736]: DEBUG oslo.service.loopingcall [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 885.581491] env[62736]: DEBUG nova.compute.manager [-] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 885.581491] env[62736]: DEBUG nova.network.neutron [-] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 885.612816] env[62736]: DEBUG nova.network.neutron [-] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 885.613339] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 2ef6b33b4837490799fff57702e82d02 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 885.621568] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ef6b33b4837490799fff57702e82d02 [ 885.682175] env[62736]: ERROR nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 885.682175] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 885.682175] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 885.682175] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 885.682175] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 885.682175] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 885.682175] env[62736]: ERROR nova.compute.manager raise self.value [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 885.682175] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 885.682175] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 885.682175] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 885.682709] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 885.682709] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 885.682709] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 885.682709] env[62736]: ERROR nova.compute.manager [ 885.682709] env[62736]: Traceback (most recent call last): [ 885.682709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 885.682709] env[62736]: listener.cb(fileno) [ 885.682709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 885.682709] env[62736]: result = function(*args, **kwargs) [ 885.682709] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 885.682709] env[62736]: return func(*args, **kwargs) [ 885.682709] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 885.682709] env[62736]: raise e [ 885.682709] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 885.682709] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 885.682709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 885.682709] env[62736]: created_port_ids = self._update_ports_for_instance( [ 885.682709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 885.682709] env[62736]: with excutils.save_and_reraise_exception(): [ 885.682709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 885.682709] env[62736]: self.force_reraise() [ 885.682709] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 885.682709] env[62736]: raise self.value [ 885.682709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 885.682709] env[62736]: updated_port = self._update_port( [ 885.682709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 885.682709] env[62736]: _ensure_no_port_binding_failure(port) [ 885.682709] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 885.682709] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 885.683462] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 885.683462] env[62736]: Removing descriptor: 16 [ 886.024164] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 886.051127] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 886.051369] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 886.051521] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 886.051700] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 886.051857] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 886.052010] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 886.052225] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 886.052379] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 886.052537] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 886.052689] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 886.052889] env[62736]: DEBUG nova.virt.hardware [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 886.053740] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11974ff5-dee6-434e-9c2e-9bdfc5eea51a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.061597] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c4464c7-9f37-4405-91ec-509c3a404e29 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.074672] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 768cf3b2-227f-463a-b192-79c50874cded has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 886.075222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 84382ed0c34645eea143f06dbfbbe7ed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 886.076648] env[62736]: ERROR nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Traceback (most recent call last): [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] yield resources [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.driver.spawn(context, instance, image_meta, [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] vm_ref = self.build_virtual_machine(instance, [ 886.076648] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] vif_infos = vmwarevif.get_vif_info(self._session, [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] for vif in network_info: [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return self._sync_wrapper(fn, *args, **kwargs) [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.wait() [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self[:] = self._gt.wait() [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return self._exit_event.wait() [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 886.076956] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] current.throw(*self._exc) [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] result = function(*args, **kwargs) [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return func(*args, **kwargs) [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise e [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] nwinfo = self.network_api.allocate_for_instance( [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] created_port_ids = self._update_ports_for_instance( [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] with excutils.save_and_reraise_exception(): [ 886.077264] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.force_reraise() [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise self.value [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] updated_port = self._update_port( [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] _ensure_no_port_binding_failure(port) [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise exception.PortBindingFailed(port_id=port['id']) [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 886.077617] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] [ 886.077617] env[62736]: INFO nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Terminating instance [ 886.080713] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 886.080898] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 886.081039] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 886.081430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 85db43467e8c456a8ced0a8f1121de8e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 886.097447] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84382ed0c34645eea143f06dbfbbe7ed [ 886.097582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 85db43467e8c456a8ced0a8f1121de8e [ 886.115354] env[62736]: DEBUG nova.network.neutron [-] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 886.115817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5390afa6cabf436c92da49da2184a92e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 886.131803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5390afa6cabf436c92da49da2184a92e [ 886.579671] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 1f1dd01d-006d-4569-8684-e1eb913ba5d4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 886.580594] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 0839a5b3642f424480423d4e0b261c81 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 886.590675] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0839a5b3642f424480423d4e0b261c81 [ 886.598269] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 886.617493] env[62736]: INFO nova.compute.manager [-] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Took 1.04 seconds to deallocate network for instance. [ 886.619612] env[62736]: DEBUG nova.compute.claims [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 886.619789] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 886.815076] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 886.815634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 17c39ab447ac4a968d9a3b3e9a9ce642 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 886.825277] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17c39ab447ac4a968d9a3b3e9a9ce642 [ 887.082637] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance c1001bab-08b9-4bb9-b737-ba7fa5f929f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 887.083260] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 12b016e4b8af467a9e74f6c7ef681b65 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.093050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12b016e4b8af467a9e74f6c7ef681b65 [ 887.319950] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 887.319950] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 887.319950] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 887.319950] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b48776b7-a15b-44c1-b982-81b59fe72bf1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.328049] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de88398e-c339-47cf-a7ca-055bd8692e62 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.342550] env[62736]: DEBUG nova.compute.manager [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Received event network-changed-57fb7c5e-9fe9-48df-ac1b-5a913e230491 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 887.342656] env[62736]: DEBUG nova.compute.manager [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Refreshing instance network info cache due to event network-changed-57fb7c5e-9fe9-48df-ac1b-5a913e230491. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 887.342862] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] Acquiring lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 887.343037] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] Acquired lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 887.343152] env[62736]: DEBUG nova.network.neutron [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Refreshing network info cache for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 887.343560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] Expecting reply to msg 98607faf9fde4c8dba18b56ebfd4d411 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.349281] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 570ee4e0-caeb-41f9-82d2-fa009f23581d could not be found. [ 887.349499] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 887.349680] env[62736]: INFO nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Took 0.03 seconds to destroy the instance on the hypervisor. [ 887.349911] env[62736]: DEBUG oslo.service.loopingcall [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 887.350116] env[62736]: DEBUG nova.compute.manager [-] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 887.350204] env[62736]: DEBUG nova.network.neutron [-] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 887.358197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98607faf9fde4c8dba18b56ebfd4d411 [ 887.364818] env[62736]: DEBUG nova.network.neutron [-] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 887.365303] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg bc7db1eca9e442da8305b33ef0a96c05 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.373153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc7db1eca9e442da8305b33ef0a96c05 [ 887.585512] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance e7d784c2-53f8-4da7-8c25-4bdf5362e94d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 887.586116] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg fe62232e35754f9fb66ce732e3a06414 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.596258] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fe62232e35754f9fb66ce732e3a06414 [ 887.867097] env[62736]: DEBUG nova.network.neutron [-] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.867514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c0e1a5da264a4088995f886689cb62b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.870814] env[62736]: DEBUG nova.network.neutron [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 887.875824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0e1a5da264a4088995f886689cb62b3 [ 887.944709] env[62736]: DEBUG nova.network.neutron [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.945370] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] Expecting reply to msg 2d5ce492d6cb4410b4aa8d801ac24549 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 887.953906] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d5ce492d6cb4410b4aa8d801ac24549 [ 888.089523] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 888.090137] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3156040f90fb41e891de53d88471a21a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 888.100331] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3156040f90fb41e891de53d88471a21a [ 888.370109] env[62736]: INFO nova.compute.manager [-] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Took 1.02 seconds to deallocate network for instance. [ 888.372473] env[62736]: DEBUG nova.compute.claims [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 888.372661] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 888.447899] env[62736]: DEBUG oslo_concurrency.lockutils [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] Releasing lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 888.448208] env[62736]: DEBUG nova.compute.manager [req-e4203cbd-9ec9-4ec3-9168-c79db881c249 req-0defdc6e-3c2e-453b-b883-19b2e140e9a6 service nova] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Received event network-vif-deleted-57fb7c5e-9fe9-48df-ac1b-5a913e230491 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 888.592466] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 6add8218-734e-4181-8ada-4657331024e5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 888.592732] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 888.592858] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1472MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 888.744996] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a019076-8b64-4e23-9399-1b4a903752f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.752806] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4bd1dbab-cc33-453d-8e2a-62e2065dfac0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.781890] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6fcc5087-008e-4e19-af7d-82f997eaaf7c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.789296] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd102468-131a-48ea-ab0d-b7f48096e820 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.803389] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 888.803942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 6a906debf7ea4e749fb37f1f64180ccd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 888.810919] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a906debf7ea4e749fb37f1f64180ccd [ 889.306623] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 889.308961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 1f3a8aa8646e454bb18f06b8b4fdef58 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 889.324913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f3a8aa8646e454bb18f06b8b4fdef58 [ 889.811777] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 889.811989] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 5.810s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 889.812283] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 21.289s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 889.813769] env[62736]: INFO nova.compute.claims [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 889.815414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 9bd173f49a3545f58f331b1e206e89b1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 889.853808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9bd173f49a3545f58f331b1e206e89b1 [ 890.320096] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg ca376faae0bc4c2da9854619fc395da6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 890.326772] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca376faae0bc4c2da9854619fc395da6 [ 890.978722] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2abfe4b4-048b-411a-9679-adbbcf280634 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 890.986162] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e9848839-cc31-424e-8389-c313a107c07f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.015645] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01bc0494-ad3c-4383-9f95-9b910430462f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.023520] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee3d898d-e6cf-46de-9aa6-0589f49879b7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.037928] env[62736]: DEBUG nova.compute.provider_tree [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 891.038585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg e66a259829b142c2a21160ad71e3ff38 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 891.046016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e66a259829b142c2a21160ad71e3ff38 [ 891.541084] env[62736]: DEBUG nova.scheduler.client.report [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 891.543386] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 344fedb5debe4ad2978f2312e116307c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 891.553963] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 344fedb5debe4ad2978f2312e116307c [ 892.046809] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.234s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 892.047316] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 892.049040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 437779594d7e432c8875d080b21839aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 892.050563] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.482s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 892.052213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg e279ce46551441868b82c108e4b17540 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 892.079734] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 437779594d7e432c8875d080b21839aa [ 892.080833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e279ce46551441868b82c108e4b17540 [ 892.555278] env[62736]: DEBUG nova.compute.utils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 892.555920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 28ccf37bd68549a8bf07c255b4cd0437 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 892.560242] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 892.560242] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 892.569971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28ccf37bd68549a8bf07c255b4cd0437 [ 892.619989] env[62736]: DEBUG nova.policy [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '52b93ec242174e5ab53c6b97f5396c6f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '06405d5d1bdb4917a235c5602d815dc5', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 892.707623] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b357b7e-ed8d-4bd5-95dc-f035e43f9bc1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.715355] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-776526e7-2aed-4f3d-b303-eb508abe00e0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.744366] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85850ce3-2c58-453f-8c54-85a6e756e1fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.751474] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60509470-02d3-497b-8b03-c85a6114ab86 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 892.764929] env[62736]: DEBUG nova.compute.provider_tree [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 892.765392] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 805fa65648a94821bc648cf1fa87ea73 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 892.772727] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 805fa65648a94821bc648cf1fa87ea73 [ 892.882812] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Successfully created port: 53a6658a-47e5-46fc-bdd1-b2efc0b47597 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 893.061122] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 893.063134] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg a663f16f38b6403890f7db3409ddf281 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.095431] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a663f16f38b6403890f7db3409ddf281 [ 893.267713] env[62736]: DEBUG nova.scheduler.client.report [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 893.270100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg b571adaaf7194d369934c0c4b63918ed in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.280379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b571adaaf7194d369934c0c4b63918ed [ 893.479360] env[62736]: DEBUG nova.compute.manager [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Received event network-changed-53a6658a-47e5-46fc-bdd1-b2efc0b47597 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 893.479644] env[62736]: DEBUG nova.compute.manager [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Refreshing instance network info cache due to event network-changed-53a6658a-47e5-46fc-bdd1-b2efc0b47597. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 893.479871] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] Acquiring lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 893.480024] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] Acquired lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 893.480185] env[62736]: DEBUG nova.network.neutron [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Refreshing network info cache for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 893.480633] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] Expecting reply to msg 88eb349d78f84ca79bd028eb1d1bacc8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.487449] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88eb349d78f84ca79bd028eb1d1bacc8 [ 893.568419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg db26df94945c445399b18da6e6b78d66 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.599155] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db26df94945c445399b18da6e6b78d66 [ 893.644010] env[62736]: ERROR nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 893.644010] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 893.644010] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 893.644010] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 893.644010] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 893.644010] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 893.644010] env[62736]: ERROR nova.compute.manager raise self.value [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 893.644010] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 893.644010] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 893.644010] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 893.644723] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 893.644723] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 893.644723] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 893.644723] env[62736]: ERROR nova.compute.manager [ 893.644723] env[62736]: Traceback (most recent call last): [ 893.644723] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 893.644723] env[62736]: listener.cb(fileno) [ 893.644723] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 893.644723] env[62736]: result = function(*args, **kwargs) [ 893.644723] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 893.644723] env[62736]: return func(*args, **kwargs) [ 893.644723] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 893.644723] env[62736]: raise e [ 893.644723] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 893.644723] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 893.644723] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 893.644723] env[62736]: created_port_ids = self._update_ports_for_instance( [ 893.644723] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 893.644723] env[62736]: with excutils.save_and_reraise_exception(): [ 893.644723] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 893.644723] env[62736]: self.force_reraise() [ 893.644723] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 893.644723] env[62736]: raise self.value [ 893.644723] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 893.644723] env[62736]: updated_port = self._update_port( [ 893.644723] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 893.644723] env[62736]: _ensure_no_port_binding_failure(port) [ 893.644723] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 893.644723] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 893.645902] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 893.645902] env[62736]: Removing descriptor: 16 [ 893.773015] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.722s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 893.773642] env[62736]: ERROR nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Traceback (most recent call last): [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.driver.spawn(context, instance, image_meta, [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self._vmops.spawn(context, instance, image_meta, injected_files, [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] vm_ref = self.build_virtual_machine(instance, [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] vif_infos = vmwarevif.get_vif_info(self._session, [ 893.773642] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] for vif in network_info: [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return self._sync_wrapper(fn, *args, **kwargs) [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.wait() [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self[:] = self._gt.wait() [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return self._exit_event.wait() [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] current.throw(*self._exc) [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 893.773968] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] result = function(*args, **kwargs) [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] return func(*args, **kwargs) [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise e [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] nwinfo = self.network_api.allocate_for_instance( [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] created_port_ids = self._update_ports_for_instance( [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] with excutils.save_and_reraise_exception(): [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] self.force_reraise() [ 893.774280] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise self.value [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] updated_port = self._update_port( [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] _ensure_no_port_binding_failure(port) [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] raise exception.PortBindingFailed(port_id=port['id']) [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] nova.exception.PortBindingFailed: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. [ 893.774647] env[62736]: ERROR nova.compute.manager [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] [ 893.774647] env[62736]: DEBUG nova.compute.utils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 893.775588] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.558s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 893.777182] env[62736]: INFO nova.compute.claims [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 893.778826] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 4b9d0625377243ffa3e7625f56ced856 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.783683] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Build of instance 7d7cc825-7dcc-4b21-bf4c-886de451863a was re-scheduled: Binding failed for port 1512881e-a7b1-4caf-adc3-f4d2e248d3cc, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 893.783683] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 893.783902] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquiring lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 893.783902] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Acquired lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 893.784021] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 893.784461] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 7a1314848e0d464193ea8f5c1d02ec9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 893.791095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a1314848e0d464193ea8f5c1d02ec9a [ 893.812894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b9d0625377243ffa3e7625f56ced856 [ 893.997138] env[62736]: DEBUG nova.network.neutron [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 894.071737] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 894.079577] env[62736]: DEBUG nova.network.neutron [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 894.079681] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] Expecting reply to msg c2d54fccb9cb4634a797052dd70df969 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.090399] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2d54fccb9cb4634a797052dd70df969 [ 894.095391] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 894.095681] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 894.095904] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 894.096163] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 894.096369] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 894.096573] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 894.096991] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 894.097221] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 894.097485] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 894.097719] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 894.097955] env[62736]: DEBUG nova.virt.hardware [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 894.099030] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13f20183-4cdd-4e46-b208-c06c3ed0a926 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.106585] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27a73dbb-c6b9-471d-986b-ddda72f6ae96 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.120043] env[62736]: ERROR nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Traceback (most recent call last): [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] yield resources [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.driver.spawn(context, instance, image_meta, [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self._vmops.spawn(context, instance, image_meta, injected_files, [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] vm_ref = self.build_virtual_machine(instance, [ 894.120043] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] vif_infos = vmwarevif.get_vif_info(self._session, [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] for vif in network_info: [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return self._sync_wrapper(fn, *args, **kwargs) [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.wait() [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self[:] = self._gt.wait() [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return self._exit_event.wait() [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 894.120465] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] current.throw(*self._exc) [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] result = function(*args, **kwargs) [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return func(*args, **kwargs) [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise e [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] nwinfo = self.network_api.allocate_for_instance( [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] created_port_ids = self._update_ports_for_instance( [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] with excutils.save_and_reraise_exception(): [ 894.120824] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.force_reraise() [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise self.value [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] updated_port = self._update_port( [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] _ensure_no_port_binding_failure(port) [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise exception.PortBindingFailed(port_id=port['id']) [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 894.121159] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] [ 894.121159] env[62736]: INFO nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Terminating instance [ 894.123650] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquiring lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 894.287885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 4c579bdf972a417285b2bdc925ea2cf4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.295756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c579bdf972a417285b2bdc925ea2cf4 [ 894.303278] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 894.373965] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 894.374507] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg e8fd0af033db47689528c241fb4b412a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.382344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8fd0af033db47689528c241fb4b412a [ 894.582014] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b4eb46a-9ce0-47ac-b5f7-b33f8fee46e1 req-2e02c7a2-0eaa-4608-b03f-98f9b8915cf8 service nova] Releasing lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 894.582443] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquired lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 894.582617] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 894.583063] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg a26031bf53f44aeb97439a638bcdcc7d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.590625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a26031bf53f44aeb97439a638bcdcc7d [ 894.876615] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Releasing lock "refresh_cache-7d7cc825-7dcc-4b21-bf4c-886de451863a" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 894.876842] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 894.877019] env[62736]: DEBUG nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 894.877179] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 894.891616] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 894.892224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg cda219f7698e45d0b974ddfe2f25c4bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.900515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cda219f7698e45d0b974ddfe2f25c4bf [ 894.934920] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ab4ce3f-d6d7-4193-bbc7-601732f192fd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.942523] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e038c08-b208-4d96-81fe-c5a6aa45d200 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.972545] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f054e53-f058-4a77-b600-a59902fa6c21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.979579] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab98566c-7c66-420d-bb49-c3d795565ffa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 894.992451] env[62736]: DEBUG nova.compute.provider_tree [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 894.992943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 44686d058f324f11975fc10cc9c32dd9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 894.999862] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44686d058f324f11975fc10cc9c32dd9 [ 895.101992] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 895.172045] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 895.172603] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg a2af53017f8e4d7fb5b95688c3acd845 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 895.180089] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2af53017f8e4d7fb5b95688c3acd845 [ 895.395324] env[62736]: DEBUG nova.network.neutron [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 895.395324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 0caa0f6fbf9743e6b28e369247a5ad86 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 895.403885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0caa0f6fbf9743e6b28e369247a5ad86 [ 895.495097] env[62736]: DEBUG nova.scheduler.client.report [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 895.497559] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 13a54da8c89e415e85147e76139d3801 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 895.507763] env[62736]: DEBUG nova.compute.manager [req-46fdbca6-0c09-4a2f-9c8a-2da226b247e2 req-17c3fb1a-30ca-40fa-b1f3-df692799615f service nova] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Received event network-vif-deleted-53a6658a-47e5-46fc-bdd1-b2efc0b47597 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 895.511836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 13a54da8c89e415e85147e76139d3801 [ 895.674747] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Releasing lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 895.675170] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 895.676030] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 895.676030] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cf92c0c0-ba3c-4c1d-ae71-14da22939608 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.686309] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ebfc423-62b1-4dff-8916-7138fd1f9672 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 895.706656] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7d047648-facb-4d58-acf7-a99a3f7adf65 could not be found. [ 895.706873] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 895.707051] env[62736]: INFO nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Took 0.03 seconds to destroy the instance on the hypervisor. [ 895.707288] env[62736]: DEBUG oslo.service.loopingcall [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 895.707515] env[62736]: DEBUG nova.compute.manager [-] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 895.707608] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 895.721415] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 895.721865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 201df4eac43e479ca31c6de2090cee27 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 895.728451] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 201df4eac43e479ca31c6de2090cee27 [ 895.898500] env[62736]: INFO nova.compute.manager [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] [instance: 7d7cc825-7dcc-4b21-bf4c-886de451863a] Took 1.02 seconds to deallocate network for instance. [ 895.900807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg d68936fb9f2c4d31b69fb9bba4f26df1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 895.934214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d68936fb9f2c4d31b69fb9bba4f26df1 [ 896.000203] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.224s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 896.000897] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 896.002700] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 67efad6732b640c6bbef818056d2eff0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.004038] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.226s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 896.005713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg be949bec97c842fc97b2b066b5acc9ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.037413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67efad6732b640c6bbef818056d2eff0 [ 896.045555] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be949bec97c842fc97b2b066b5acc9ea [ 896.223900] env[62736]: DEBUG nova.network.neutron [-] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 896.224396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5f3e6062445341cebc347bde811f34f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.232800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5f3e6062445341cebc347bde811f34f0 [ 896.405628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg 3d2aecf950ef46cfb8d403ef1e230a44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.434229] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d2aecf950ef46cfb8d403ef1e230a44 [ 896.508622] env[62736]: DEBUG nova.compute.utils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 896.509151] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 954ca9f61b93479fb431fa3502520f04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.510120] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 896.510235] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 896.521412] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 954ca9f61b93479fb431fa3502520f04 [ 896.561585] env[62736]: DEBUG nova.policy [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6a6d2a917fec48e48e943137dff043fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd781d0a045444391bc070520be1b6431', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 896.658492] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46834e44-caee-4b31-90e7-908953f60f2d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.665939] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29a3329a-0779-4f7c-85e3-5388dbafd946 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.696761] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f4200d8-3d1f-486a-9ba1-0903d3f37ce5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.703656] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d94f418-e53e-4aa8-9b49-9656f53fc1c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 896.718535] env[62736]: DEBUG nova.compute.provider_tree [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 896.719117] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg d1a0f2053cfb4133aeed9feba325c2aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.726646] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1a0f2053cfb4133aeed9feba325c2aa [ 896.727238] env[62736]: INFO nova.compute.manager [-] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Took 1.02 seconds to deallocate network for instance. [ 896.729232] env[62736]: DEBUG nova.compute.claims [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 896.729468] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 896.841837] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Successfully created port: 343f61a7-4e93-40f6-87ad-1e52b67a570e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 896.928055] env[62736]: INFO nova.scheduler.client.report [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Deleted allocations for instance 7d7cc825-7dcc-4b21-bf4c-886de451863a [ 896.934969] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Expecting reply to msg cd2326ab61bf46848ab7565bacfc296a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 896.952728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cd2326ab61bf46848ab7565bacfc296a [ 897.013324] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 897.016318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 781b036ec5f84f5ab95f56b29f559770 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.057607] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 781b036ec5f84f5ab95f56b29f559770 [ 897.222355] env[62736]: DEBUG nova.scheduler.client.report [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 897.224707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg d2b8567e6c0842408c06ea83f8c4ff08 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.235146] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2b8567e6c0842408c06ea83f8c4ff08 [ 897.442382] env[62736]: DEBUG oslo_concurrency.lockutils [None req-34660ada-fb48-4513-af27-bf59e1bb698a tempest-ServerAddressesTestJSON-1247546411 tempest-ServerAddressesTestJSON-1247546411-project-member] Lock "7d7cc825-7dcc-4b21-bf4c-886de451863a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 83.742s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 897.443552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 003bfaceca1047019bf5c67492d01e39 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.453596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 003bfaceca1047019bf5c67492d01e39 [ 897.521465] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg b8534e43a6234048b74c7bc4982765d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.551475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b8534e43a6234048b74c7bc4982765d9 [ 897.668639] env[62736]: ERROR nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 897.668639] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.668639] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.668639] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.668639] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.668639] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.668639] env[62736]: ERROR nova.compute.manager raise self.value [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.668639] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 897.668639] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.668639] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 897.669361] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.669361] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 897.669361] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 897.669361] env[62736]: ERROR nova.compute.manager [ 897.669361] env[62736]: Traceback (most recent call last): [ 897.669361] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 897.669361] env[62736]: listener.cb(fileno) [ 897.669361] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 897.669361] env[62736]: result = function(*args, **kwargs) [ 897.669361] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 897.669361] env[62736]: return func(*args, **kwargs) [ 897.669361] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 897.669361] env[62736]: raise e [ 897.669361] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.669361] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 897.669361] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.669361] env[62736]: created_port_ids = self._update_ports_for_instance( [ 897.669361] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.669361] env[62736]: with excutils.save_and_reraise_exception(): [ 897.669361] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.669361] env[62736]: self.force_reraise() [ 897.669361] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.669361] env[62736]: raise self.value [ 897.669361] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.669361] env[62736]: updated_port = self._update_port( [ 897.669361] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.669361] env[62736]: _ensure_no_port_binding_failure(port) [ 897.669361] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.669361] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 897.670773] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 897.670773] env[62736]: Removing descriptor: 16 [ 897.729465] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.723s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 897.729465] env[62736]: ERROR nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Traceback (most recent call last): [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.driver.spawn(context, instance, image_meta, [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 897.729465] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] vm_ref = self.build_virtual_machine(instance, [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] vif_infos = vmwarevif.get_vif_info(self._session, [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] for vif in network_info: [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return self._sync_wrapper(fn, *args, **kwargs) [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.wait() [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self[:] = self._gt.wait() [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return self._exit_event.wait() [ 897.732602] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] current.throw(*self._exc) [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] result = function(*args, **kwargs) [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] return func(*args, **kwargs) [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise e [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] nwinfo = self.network_api.allocate_for_instance( [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] created_port_ids = self._update_ports_for_instance( [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 897.733397] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] with excutils.save_and_reraise_exception(): [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] self.force_reraise() [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise self.value [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] updated_port = self._update_port( [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] _ensure_no_port_binding_failure(port) [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] raise exception.PortBindingFailed(port_id=port['id']) [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] nova.exception.PortBindingFailed: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. [ 897.734199] env[62736]: ERROR nova.compute.manager [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] [ 897.734834] env[62736]: DEBUG nova.compute.utils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 897.734834] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.243s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 897.734834] env[62736]: INFO nova.compute.claims [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 897.734834] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 54d2a7a38bcc461d81a914ef698f8afa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.735058] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Build of instance 86be55c9-82d5-4a34-b628-4729c42d83c1 was re-scheduled: Binding failed for port 7f9f039c-f6fc-4bc1-a2fa-317383f60164, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 897.735058] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 897.735058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquiring lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 897.735058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Acquired lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 897.735312] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 897.735551] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg ac2ee07fda4443929fac67e7877628ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.743343] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ac2ee07fda4443929fac67e7877628ca [ 897.744722] env[62736]: DEBUG nova.compute.manager [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Received event network-changed-343f61a7-4e93-40f6-87ad-1e52b67a570e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 897.744899] env[62736]: DEBUG nova.compute.manager [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Refreshing instance network info cache due to event network-changed-343f61a7-4e93-40f6-87ad-1e52b67a570e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 897.745101] env[62736]: DEBUG oslo_concurrency.lockutils [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] Acquiring lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 897.745273] env[62736]: DEBUG oslo_concurrency.lockutils [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] Acquired lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 897.745434] env[62736]: DEBUG nova.network.neutron [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Refreshing network info cache for port 343f61a7-4e93-40f6-87ad-1e52b67a570e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 897.745799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] Expecting reply to msg 022873904c0f4a4eb2ba0205218bba3d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.754509] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 022873904c0f4a4eb2ba0205218bba3d [ 897.774885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54d2a7a38bcc461d81a914ef698f8afa [ 897.945186] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 897.946996] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg c7f3009cf4d34255b51a1365ecb85c68 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 897.984927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c7f3009cf4d34255b51a1365ecb85c68 [ 898.024898] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 898.050153] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 898.050402] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 898.050553] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 898.050746] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 898.050888] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 898.051029] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 898.051230] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 898.051385] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 898.051549] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 898.051743] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 898.051918] env[62736]: DEBUG nova.virt.hardware [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 898.053064] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6691dfa2-f6bf-4de6-b7b9-5037d27f093c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.060940] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90065a92-06d4-4a2e-b441-831f1df24f95 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.075002] env[62736]: ERROR nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Traceback (most recent call last): [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] yield resources [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.driver.spawn(context, instance, image_meta, [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self._vmops.spawn(context, instance, image_meta, injected_files, [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] vm_ref = self.build_virtual_machine(instance, [ 898.075002] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] vif_infos = vmwarevif.get_vif_info(self._session, [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] for vif in network_info: [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return self._sync_wrapper(fn, *args, **kwargs) [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.wait() [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self[:] = self._gt.wait() [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return self._exit_event.wait() [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 898.075359] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] current.throw(*self._exc) [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] result = function(*args, **kwargs) [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return func(*args, **kwargs) [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise e [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] nwinfo = self.network_api.allocate_for_instance( [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] created_port_ids = self._update_ports_for_instance( [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] with excutils.save_and_reraise_exception(): [ 898.075690] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.force_reraise() [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise self.value [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] updated_port = self._update_port( [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] _ensure_no_port_binding_failure(port) [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise exception.PortBindingFailed(port_id=port['id']) [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 898.076088] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] [ 898.076088] env[62736]: INFO nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Terminating instance [ 898.077418] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 898.241274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg f3bb006feb9c4d0f892a389dd7398b5a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 898.249691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3bb006feb9c4d0f892a389dd7398b5a [ 898.261530] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 898.278912] env[62736]: DEBUG nova.network.neutron [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 898.367191] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 898.367720] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 072826f3ec8f43a3a7d2f99529d2cd0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 898.376902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 072826f3ec8f43a3a7d2f99529d2cd0a [ 898.392632] env[62736]: DEBUG nova.network.neutron [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 898.393283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] Expecting reply to msg 4a6bdb1eaee1427a919c05d74997c580 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 898.402352] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a6bdb1eaee1427a919c05d74997c580 [ 898.469401] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.870053] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Releasing lock "refresh_cache-86be55c9-82d5-4a34-b628-4729c42d83c1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 898.870694] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 898.871162] env[62736]: DEBUG nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 898.871735] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 898.893424] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 898.894009] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg f5b8f8445e4c489287fd116ceb7393fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 898.901151] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5b8f8445e4c489287fd116ceb7393fd [ 898.902851] env[62736]: DEBUG oslo_concurrency.lockutils [req-fcf1072a-fa86-4970-a562-4211ec5cd0eb req-ffd6a667-4a2d-4ccd-8daa-0142129b1755 service nova] Releasing lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 898.903432] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 898.903611] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 898.904024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg b4cd3940e38e420cb3cba5380a7479bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 898.913196] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b4cd3940e38e420cb3cba5380a7479bd [ 898.988941] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47cda5b3-0a2f-4f49-8f8b-80e226ac7eda {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.997348] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-440b253b-7af9-42eb-bd7f-25776e2bd786 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.030045] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-496ffabf-71ea-4796-b104-02785a14f47a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.039202] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0dbf79a-8f95-4e66-b566-a84b7ac44518 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.056154] env[62736]: DEBUG nova.compute.provider_tree [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 899.056950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 562b30b04be1413dba9d680a98d65ddb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 899.066095] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 562b30b04be1413dba9d680a98d65ddb [ 899.396200] env[62736]: DEBUG nova.network.neutron [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 899.396757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 30c2100d9cf045d792b1a17ac43d404d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 899.404899] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30c2100d9cf045d792b1a17ac43d404d [ 899.420617] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 899.495813] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 899.496381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 017f8952269144f8a8c4724fe0bced14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 899.504165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 017f8952269144f8a8c4724fe0bced14 [ 899.560239] env[62736]: DEBUG nova.scheduler.client.report [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 899.562531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 8a877779befc4159839f25035f3ec35f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 899.572837] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a877779befc4159839f25035f3ec35f [ 899.769596] env[62736]: DEBUG nova.compute.manager [req-9e74c829-541a-4f13-a4eb-502d4200df3a req-20f4f40b-e9d1-47fb-aa55-6a0f7915161e service nova] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Received event network-vif-deleted-343f61a7-4e93-40f6-87ad-1e52b67a570e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 899.901457] env[62736]: INFO nova.compute.manager [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] [instance: 86be55c9-82d5-4a34-b628-4729c42d83c1] Took 1.03 seconds to deallocate network for instance. [ 899.903213] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 0d802bc66afb4407aaaa9cb2b81846ce in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 899.937027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0d802bc66afb4407aaaa9cb2b81846ce [ 899.998831] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 899.999274] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 899.999532] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 899.999815] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8cbdcdb4-42e2-459b-b8ea-574a8aa7bbb3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.008733] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d119a52-e34e-410a-b2bd-7d4e41b68338 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 900.029167] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fa27cfe2-6f7f-4397-bcc5-147d6865f404 could not be found. [ 900.029371] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 900.029610] env[62736]: INFO nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Took 0.03 seconds to destroy the instance on the hypervisor. [ 900.029863] env[62736]: DEBUG oslo.service.loopingcall [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 900.030075] env[62736]: DEBUG nova.compute.manager [-] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 900.030163] env[62736]: DEBUG nova.network.neutron [-] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 900.050592] env[62736]: DEBUG nova.network.neutron [-] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 900.051138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 12f1f03fec8743c6a32ecc6638ead478 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.057992] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12f1f03fec8743c6a32ecc6638ead478 [ 900.064672] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.335s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 900.065007] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 65e4299d21024097888ad3198daed3f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.065757] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 20.012s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.067173] env[62736]: INFO nova.compute.claims [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 900.069030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 96501c185ef64ca38e5704065f791bf9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.079008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 65e4299d21024097888ad3198daed3f1 [ 900.102959] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96501c185ef64ca38e5704065f791bf9 [ 900.408148] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg 0a4583510a134320805466190bd4f2f7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.435874] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a4583510a134320805466190bd4f2f7 [ 900.552896] env[62736]: DEBUG nova.network.neutron [-] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 900.553369] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 49807df2019a443a9c7eaa93de4d8a4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.562239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49807df2019a443a9c7eaa93de4d8a4d [ 900.570922] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "1b1ff1ec-5e0c-48fb-8707-90f40ff05fc8" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 900.571150] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "1b1ff1ec-5e0c-48fb-8707-90f40ff05fc8" acquired by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 900.571585] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 7f43b4d3f7674b9aac5d35c06f4b6db8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.573483] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a7f3ce257e654a51899404c96c252b3d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.577905] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f43b4d3f7674b9aac5d35c06f4b6db8 [ 900.579517] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7f3ce257e654a51899404c96c252b3d [ 900.933994] env[62736]: INFO nova.scheduler.client.report [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Deleted allocations for instance 86be55c9-82d5-4a34-b628-4729c42d83c1 [ 900.940049] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Expecting reply to msg ffb681c7eac14e6cb10705896c8b471d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 900.955788] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ffb681c7eac14e6cb10705896c8b471d [ 901.055524] env[62736]: INFO nova.compute.manager [-] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Took 1.03 seconds to deallocate network for instance. [ 901.057923] env[62736]: DEBUG nova.compute.claims [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 901.058098] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 901.074667] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "1b1ff1ec-5e0c-48fb-8707-90f40ff05fc8" "released" by "nova.compute.manager.ComputeManager._validate_instance_group_policy.._do_validation" :: held 0.503s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 901.075173] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 901.076816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 05143df3352448dca5cb049dad39d0ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.107656] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 05143df3352448dca5cb049dad39d0ff [ 901.207364] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b8a3b2e-a8c8-46bc-ba51-2d41b36af671 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.214601] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-451f0182-440e-47dc-ba34-63558422085f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.244651] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d1196e0-3e65-43cc-888b-86e7bdc3e7dd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.251672] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e78743d6-4564-42a5-a65e-0ecb3b83f2ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 901.264391] env[62736]: DEBUG nova.compute.provider_tree [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 901.264875] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg aa6f9ebbbce44e128b2980072a02b91a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.271350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa6f9ebbbce44e128b2980072a02b91a [ 901.445924] env[62736]: DEBUG oslo_concurrency.lockutils [None req-8282e491-1f8c-44fe-8756-ffd5bf3244f3 tempest-ServerMetadataNegativeTestJSON-1390807253 tempest-ServerMetadataNegativeTestJSON-1390807253-project-member] Lock "86be55c9-82d5-4a34-b628-4729c42d83c1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.951s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 901.446526] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg b9bdd0e2135f41d78155e8a38059f9fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.456348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9bdd0e2135f41d78155e8a38059f9fe [ 901.582591] env[62736]: DEBUG nova.compute.utils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 901.583245] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg e3c2a5dc880545c78f69adee66ce7418 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.584185] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 901.584363] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 901.592551] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e3c2a5dc880545c78f69adee66ce7418 [ 901.654857] env[62736]: DEBUG nova.policy [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cf79d14f7faf42139e1e8f342e1fe9b6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cc9bb22c5b864296a51086af6ddd40d7', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 901.767061] env[62736]: DEBUG nova.scheduler.client.report [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 901.769828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b248dfc5380c4cb28b93d4efc8b2cd01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.780704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b248dfc5380c4cb28b93d4efc8b2cd01 [ 901.948874] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 901.951021] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 5e7d67d2765d4846aa2775130f3b8c09 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 901.981459] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e7d67d2765d4846aa2775130f3b8c09 [ 902.087013] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 902.090801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 575ae55069374446b675a808517eea75 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 902.131061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 575ae55069374446b675a808517eea75 [ 902.272669] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.207s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 902.273433] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 902.275285] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 44418ac339df4ff4b753caae361a3b30 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 902.276991] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.761s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 902.279023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg a6b42b3796e5468ea8f62d5906cb16e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 902.314038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44418ac339df4ff4b753caae361a3b30 [ 902.316467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6b42b3796e5468ea8f62d5906cb16e0 [ 902.370170] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Successfully created port: 68d97bd5-2b32-4ce1-bb1d-5d90628a1951 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 902.470520] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 902.596102] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 36a4bff1ca574f5d82e6d1772cd66cd2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 902.627711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36a4bff1ca574f5d82e6d1772cd66cd2 [ 902.782156] env[62736]: DEBUG nova.compute.utils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 902.782798] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a2ed4a0687024b8890787c2a0e4dfefb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 902.786713] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 902.786878] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 902.794614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2ed4a0687024b8890787c2a0e4dfefb [ 902.850298] env[62736]: DEBUG nova.policy [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 902.959549] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdc9b001-5829-42cf-bb06-791c3c618e10 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 902.967412] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebc55978-ce6a-4920-8ef4-aeed05389a1a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.000380] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9f511074-4a88-4496-bcd1-2de1c06063af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.007487] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0f38e9c-cf77-45b1-b79d-a4f6c21fb9d2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.021672] env[62736]: DEBUG nova.compute.provider_tree [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 903.022219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 36fa8b41a3424eb1b2f1ecb426d09bfd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 903.030263] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36fa8b41a3424eb1b2f1ecb426d09bfd [ 903.099213] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 903.127494] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 903.127746] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 903.127898] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 903.128086] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 903.128229] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 903.128474] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 903.128598] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 903.128708] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 903.129077] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 903.129077] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 903.129217] env[62736]: DEBUG nova.virt.hardware [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 903.130025] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b940619e-38e3-4293-ae7c-87817907f810 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.144186] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d43aa6aa-64d6-46a7-b697-36d663724e08 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 903.165491] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Successfully created port: a06f0a41-bdaf-4090-bfe5-9dc6efaf2200 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 903.290180] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 903.292161] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 3eb20b40b4ef442398a0ba34b3f577b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 903.330596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3eb20b40b4ef442398a0ba34b3f577b3 [ 903.528923] env[62736]: DEBUG nova.scheduler.client.report [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 903.528923] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg d492aba79e5b41d8afcb051501b6b99f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 903.540638] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d492aba79e5b41d8afcb051501b6b99f [ 903.653830] env[62736]: DEBUG nova.compute.manager [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Received event network-changed-68d97bd5-2b32-4ce1-bb1d-5d90628a1951 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 903.653830] env[62736]: DEBUG nova.compute.manager [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Refreshing instance network info cache due to event network-changed-68d97bd5-2b32-4ce1-bb1d-5d90628a1951. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 903.653830] env[62736]: DEBUG oslo_concurrency.lockutils [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] Acquiring lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 903.653830] env[62736]: DEBUG oslo_concurrency.lockutils [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] Acquired lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 903.653830] env[62736]: DEBUG nova.network.neutron [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Refreshing network info cache for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 903.654195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] Expecting reply to msg 99fc2f0bac6c46a2b5147ec3d530bcb3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 903.660273] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99fc2f0bac6c46a2b5147ec3d530bcb3 [ 903.802280] env[62736]: ERROR nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 903.802280] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 903.802280] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 903.802280] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 903.802280] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 903.802280] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 903.802280] env[62736]: ERROR nova.compute.manager raise self.value [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 903.802280] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 903.802280] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 903.802280] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 903.802786] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 903.802786] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 903.802786] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 903.802786] env[62736]: ERROR nova.compute.manager [ 903.802786] env[62736]: Traceback (most recent call last): [ 903.802786] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 903.802786] env[62736]: listener.cb(fileno) [ 903.802786] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 903.802786] env[62736]: result = function(*args, **kwargs) [ 903.802786] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 903.802786] env[62736]: return func(*args, **kwargs) [ 903.802786] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 903.802786] env[62736]: raise e [ 903.802786] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 903.802786] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 903.802786] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 903.802786] env[62736]: created_port_ids = self._update_ports_for_instance( [ 903.802786] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 903.802786] env[62736]: with excutils.save_and_reraise_exception(): [ 903.802786] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 903.802786] env[62736]: self.force_reraise() [ 903.802786] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 903.802786] env[62736]: raise self.value [ 903.802786] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 903.802786] env[62736]: updated_port = self._update_port( [ 903.802786] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 903.802786] env[62736]: _ensure_no_port_binding_failure(port) [ 903.802786] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 903.802786] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 903.803645] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 903.803645] env[62736]: Removing descriptor: 17 [ 903.803645] env[62736]: ERROR nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] Traceback (most recent call last): [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] yield resources [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.driver.spawn(context, instance, image_meta, [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self._vmops.spawn(context, instance, image_meta, injected_files, [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 903.803645] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] vm_ref = self.build_virtual_machine(instance, [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] vif_infos = vmwarevif.get_vif_info(self._session, [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] for vif in network_info: [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self._sync_wrapper(fn, *args, **kwargs) [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.wait() [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self[:] = self._gt.wait() [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self._exit_event.wait() [ 903.803997] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] result = hub.switch() [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self.greenlet.switch() [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] result = function(*args, **kwargs) [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return func(*args, **kwargs) [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise e [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] nwinfo = self.network_api.allocate_for_instance( [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 903.804391] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] created_port_ids = self._update_ports_for_instance( [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] with excutils.save_and_reraise_exception(): [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.force_reraise() [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise self.value [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] updated_port = self._update_port( [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] _ensure_no_port_binding_failure(port) [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 903.804762] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise exception.PortBindingFailed(port_id=port['id']) [ 903.805148] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 903.805148] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] [ 903.805148] env[62736]: INFO nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Terminating instance [ 903.805506] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 903.809358] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 10ff97ea1cab4d0d958270e82652780b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 903.844605] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10ff97ea1cab4d0d958270e82652780b [ 904.020442] env[62736]: DEBUG nova.compute.manager [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Received event network-changed-a06f0a41-bdaf-4090-bfe5-9dc6efaf2200 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 904.020716] env[62736]: DEBUG nova.compute.manager [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Refreshing instance network info cache due to event network-changed-a06f0a41-bdaf-4090-bfe5-9dc6efaf2200. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 904.020956] env[62736]: DEBUG oslo_concurrency.lockutils [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] Acquiring lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.021111] env[62736]: DEBUG oslo_concurrency.lockutils [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] Acquired lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.021284] env[62736]: DEBUG nova.network.neutron [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Refreshing network info cache for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 904.022697] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] Expecting reply to msg 3df9d9fa12e64067aad2d1284440c74c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.028945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3df9d9fa12e64067aad2d1284440c74c [ 904.030602] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.754s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 904.031194] env[62736]: ERROR nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Traceback (most recent call last): [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.driver.spawn(context, instance, image_meta, [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self._vmops.spawn(context, instance, image_meta, injected_files, [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] vm_ref = self.build_virtual_machine(instance, [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] vif_infos = vmwarevif.get_vif_info(self._session, [ 904.031194] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] for vif in network_info: [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return self._sync_wrapper(fn, *args, **kwargs) [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.wait() [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self[:] = self._gt.wait() [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return self._exit_event.wait() [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] current.throw(*self._exc) [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 904.031491] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] result = function(*args, **kwargs) [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] return func(*args, **kwargs) [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise e [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] nwinfo = self.network_api.allocate_for_instance( [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] created_port_ids = self._update_ports_for_instance( [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] with excutils.save_and_reraise_exception(): [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] self.force_reraise() [ 904.031802] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise self.value [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] updated_port = self._update_port( [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] _ensure_no_port_binding_failure(port) [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] raise exception.PortBindingFailed(port_id=port['id']) [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] nova.exception.PortBindingFailed: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. [ 904.032164] env[62736]: ERROR nova.compute.manager [instance: 53625c0d-203a-45f0-b899-5d2673413a53] [ 904.032164] env[62736]: DEBUG nova.compute.utils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 904.034017] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Build of instance 53625c0d-203a-45f0-b899-5d2673413a53 was re-scheduled: Binding failed for port 7b9d929d-0980-4cd6-ad6b-83398958648e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 904.034422] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 904.034647] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquiring lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.034789] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Acquired lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.034940] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 904.035315] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg a994832bb886417fab2f995d38cefdaa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.036565] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 19.757s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 904.038189] env[62736]: INFO nova.compute.claims [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 904.039802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 7a9a2574ac104030866967edc93836d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.049109] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a994832bb886417fab2f995d38cefdaa [ 904.078749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a9a2574ac104030866967edc93836d0 [ 904.184880] env[62736]: DEBUG nova.network.neutron [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.211350] env[62736]: ERROR nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 904.211350] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.211350] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 904.211350] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 904.211350] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.211350] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.211350] env[62736]: ERROR nova.compute.manager raise self.value [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 904.211350] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 904.211350] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.211350] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 904.211805] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.211805] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 904.211805] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 904.211805] env[62736]: ERROR nova.compute.manager [ 904.211805] env[62736]: Traceback (most recent call last): [ 904.211805] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 904.211805] env[62736]: listener.cb(fileno) [ 904.211805] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 904.211805] env[62736]: result = function(*args, **kwargs) [ 904.211805] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 904.211805] env[62736]: return func(*args, **kwargs) [ 904.211805] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 904.211805] env[62736]: raise e [ 904.211805] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.211805] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 904.211805] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 904.211805] env[62736]: created_port_ids = self._update_ports_for_instance( [ 904.211805] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 904.211805] env[62736]: with excutils.save_and_reraise_exception(): [ 904.211805] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.211805] env[62736]: self.force_reraise() [ 904.211805] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.211805] env[62736]: raise self.value [ 904.211805] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 904.211805] env[62736]: updated_port = self._update_port( [ 904.211805] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.211805] env[62736]: _ensure_no_port_binding_failure(port) [ 904.211805] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.211805] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 904.212758] env[62736]: nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 904.212758] env[62736]: Removing descriptor: 16 [ 904.286821] env[62736]: DEBUG nova.network.neutron [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.287394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] Expecting reply to msg 0fa67dd8c0004f438dbcbde565c7af56 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.295799] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fa67dd8c0004f438dbcbde565c7af56 [ 904.318096] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 904.348828] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 904.348828] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 904.348828] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 904.348828] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 904.349037] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 904.349037] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 904.349037] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 904.349186] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 904.349337] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 904.349523] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 904.349747] env[62736]: DEBUG nova.virt.hardware [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 904.350575] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d42bad6-cd83-4556-a653-eb5ffb349d0d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.358862] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-164f72d1-a372-4c58-bd85-96479a0b0fa4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 904.373892] env[62736]: ERROR nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Traceback (most recent call last): [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] yield resources [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.driver.spawn(context, instance, image_meta, [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] vm_ref = self.build_virtual_machine(instance, [ 904.373892] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] vif_infos = vmwarevif.get_vif_info(self._session, [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] for vif in network_info: [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return self._sync_wrapper(fn, *args, **kwargs) [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.wait() [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self[:] = self._gt.wait() [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return self._exit_event.wait() [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 904.374228] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] current.throw(*self._exc) [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] result = function(*args, **kwargs) [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return func(*args, **kwargs) [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise e [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] nwinfo = self.network_api.allocate_for_instance( [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] created_port_ids = self._update_ports_for_instance( [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] with excutils.save_and_reraise_exception(): [ 904.374629] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.force_reraise() [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise self.value [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] updated_port = self._update_port( [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] _ensure_no_port_binding_failure(port) [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise exception.PortBindingFailed(port_id=port['id']) [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 904.375065] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] [ 904.375065] env[62736]: INFO nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Terminating instance [ 904.376549] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 904.536964] env[62736]: DEBUG nova.network.neutron [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.543301] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 64ec582a74544b75a92f83d74d377105 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.550850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64ec582a74544b75a92f83d74d377105 [ 904.581949] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.605265] env[62736]: DEBUG nova.network.neutron [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.605779] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] Expecting reply to msg 0a9241b772f343d783fe3ae7e0118714 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.614190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a9241b772f343d783fe3ae7e0118714 [ 904.707553] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.708077] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 93fe69ca7e804b5bb5de4deeeb256366 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.716578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93fe69ca7e804b5bb5de4deeeb256366 [ 904.789295] env[62736]: DEBUG oslo_concurrency.lockutils [req-c84983f2-e6a3-4e85-bf18-345283342929 req-4e67464c-0aa3-408e-9e77-547c5c63d37c service nova] Releasing lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 904.789831] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquired lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 904.790017] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 904.790472] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 64926610ae5d4fda86c3bb39105123c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 904.798422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64926610ae5d4fda86c3bb39105123c6 [ 905.107602] env[62736]: DEBUG oslo_concurrency.lockutils [req-53941ae8-af58-4bd9-8e03-272a3df32204 req-2355508b-6f4e-44c0-b553-0d142b1f4f96 service nova] Releasing lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.108010] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 905.108339] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 905.108640] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 86951c823e314a04a3dfb1852e5ee252 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.115597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86951c823e314a04a3dfb1852e5ee252 [ 905.199519] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0674cff-f377-476d-b583-89eb49fe6127 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.207494] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce2c6821-3be3-4f63-86ed-faf70b8d8ffc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.210731] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Releasing lock "refresh_cache-53625c0d-203a-45f0-b899-5d2673413a53" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 905.210945] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 905.211122] env[62736]: DEBUG nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 905.211286] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 905.239326] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 905.239998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg a4044cc0c9264d6d9b4fcd336a501ccc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.241370] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8563ebfc-5ce7-44da-8985-10abbfaf05c0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.246832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4044cc0c9264d6d9b4fcd336a501ccc [ 905.250445] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1276e201-363d-4f9e-b764-f675ca6f74f6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 905.264085] env[62736]: DEBUG nova.compute.provider_tree [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 905.264579] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 40c45f8b95764f0193973cb84423ed71 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.271553] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40c45f8b95764f0193973cb84423ed71 [ 905.318394] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 905.591059] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.591590] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 134f9e59b3d84bc4aa85b47529eff59b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.599773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 134f9e59b3d84bc4aa85b47529eff59b [ 905.631594] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 905.677108] env[62736]: DEBUG nova.compute.manager [req-fa47268b-d0c7-40b7-bc01-64a17ca02d35 req-1e588691-46d9-4cbd-91ea-7ab1d6b29ddc service nova] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Received event network-vif-deleted-68d97bd5-2b32-4ce1-bb1d-5d90628a1951 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 905.746496] env[62736]: DEBUG nova.network.neutron [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.747200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg b1562c0a679d43e5a6f178b93bf1e16d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.760578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1562c0a679d43e5a6f178b93bf1e16d [ 905.762327] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.763186] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d8e5aeacedd546a792b1e096e769133a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.767201] env[62736]: DEBUG nova.scheduler.client.report [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 905.772130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 4a9091f2f643472ab5926caa1abc2b14 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 905.774006] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8e5aeacedd546a792b1e096e769133a [ 905.784309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4a9091f2f643472ab5926caa1abc2b14 [ 906.057784] env[62736]: DEBUG nova.compute.manager [req-e95341a9-d24d-4183-b421-6c962fd80f7e req-af1aca54-b440-4e14-b7d6-73e825b0d015 service nova] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Received event network-vif-deleted-a06f0a41-bdaf-4090-bfe5-9dc6efaf2200 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 906.094162] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Releasing lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 906.094570] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 906.094772] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 906.095441] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9d972b49-a299-42cd-8256-28e2f6d12acf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.104673] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e3ec9acb-6589-4abc-a642-36ec99409029 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.126074] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 768cf3b2-227f-463a-b192-79c50874cded could not be found. [ 906.126321] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 906.126472] env[62736]: INFO nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Took 0.03 seconds to destroy the instance on the hypervisor. [ 906.126715] env[62736]: DEBUG oslo.service.loopingcall [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 906.126937] env[62736]: DEBUG nova.compute.manager [-] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 906.127031] env[62736]: DEBUG nova.network.neutron [-] [instance: 768cf3b2-227f-463a-b192-79c50874cded] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 906.141921] env[62736]: DEBUG nova.network.neutron [-] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 906.142389] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4826ec7f63e6479bbfe9861128661b29 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.149035] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4826ec7f63e6479bbfe9861128661b29 [ 906.250203] env[62736]: INFO nova.compute.manager [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] [instance: 53625c0d-203a-45f0-b899-5d2673413a53] Took 1.04 seconds to deallocate network for instance. [ 906.252021] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 51817a1185a84f3c9c9a57b83813d523 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.265801] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 906.266188] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 906.266371] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 906.267791] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f748c02c-b826-41af-86e0-1494ed0d78a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.275145] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9215d604-e9c9-478c-9f08-f44e0fd0237a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.286565] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.250s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.287034] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 906.288653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 549b6e5a53354ef79390cf2dd9e6fb89 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.290111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51817a1185a84f3c9c9a57b83813d523 [ 906.290679] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.671s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.292335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 9e2e2009b3324166ba4a726c8654748c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.304124] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1f1dd01d-006d-4569-8684-e1eb913ba5d4 could not be found. [ 906.304344] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 906.304517] env[62736]: INFO nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Took 0.04 seconds to destroy the instance on the hypervisor. [ 906.304751] env[62736]: DEBUG oslo.service.loopingcall [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 906.304962] env[62736]: DEBUG nova.compute.manager [-] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 906.305053] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 906.324258] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 906.324795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 959ea24f86b245ccb1d0261f42b10e8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.326467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 549b6e5a53354ef79390cf2dd9e6fb89 [ 906.329904] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e2e2009b3324166ba4a726c8654748c [ 906.330303] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 959ea24f86b245ccb1d0261f42b10e8b [ 906.644505] env[62736]: DEBUG nova.network.neutron [-] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 906.645014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3180e17ddee2430f94a8c3cafaad8804 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.653231] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3180e17ddee2430f94a8c3cafaad8804 [ 906.756836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg 362f7f763157440598d8f4e2d70d98aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.785992] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 362f7f763157440598d8f4e2d70d98aa [ 906.792160] env[62736]: DEBUG nova.compute.utils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 906.792791] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 505c2313bb404ac69e62245bd879ebf9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.793632] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 906.793791] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 906.804221] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 505c2313bb404ac69e62245bd879ebf9 [ 906.827017] env[62736]: DEBUG nova.network.neutron [-] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 906.827569] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 24ff1cefb6e64a138b417ddda2bdbb07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 906.838324] env[62736]: DEBUG nova.policy [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 906.840440] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24ff1cefb6e64a138b417ddda2bdbb07 [ 906.957980] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55af97e4-299e-40b4-893d-2afd9e0fa8dc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.963333] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b55abceb-5da6-4b6d-8553-248390696d84 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 906.994604] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1fec252-d0b1-4dfa-a132-056c6a622af3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.002413] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e207f17-e89e-4542-a91e-87ef96ada1cb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 907.015347] env[62736]: DEBUG nova.compute.provider_tree [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 907.015952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 4715ba731014488bbce07c527d34f963 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.025530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4715ba731014488bbce07c527d34f963 [ 907.112141] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquiring lock "41e8e7c0-a518-43bc-82fa-5f77704f4fc1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.112390] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "41e8e7c0-a518-43bc-82fa-5f77704f4fc1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 907.139580] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Successfully created port: f4444a92-d000-41ce-be26-294346c5047b {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 907.146511] env[62736]: INFO nova.compute.manager [-] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Took 1.02 seconds to deallocate network for instance. [ 907.148650] env[62736]: DEBUG nova.compute.claims [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 907.148822] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.288332] env[62736]: INFO nova.scheduler.client.report [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Deleted allocations for instance 53625c0d-203a-45f0-b899-5d2673413a53 [ 907.294003] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Expecting reply to msg dc653f6272ef408bb1bdfc02a11191e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.296255] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 907.297725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1697b0fc6c8b4ab89bb9c070c0c4a24c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.307529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc653f6272ef408bb1bdfc02a11191e7 [ 907.333055] env[62736]: INFO nova.compute.manager [-] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Took 1.03 seconds to deallocate network for instance. [ 907.340707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1697b0fc6c8b4ab89bb9c070c0c4a24c [ 907.341526] env[62736]: DEBUG nova.compute.claims [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 907.341709] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 907.518368] env[62736]: DEBUG nova.scheduler.client.report [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 907.520795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg a43dc2d1a52b457f980c97d9ac9cdd26 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.533318] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a43dc2d1a52b457f980c97d9ac9cdd26 [ 907.801519] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a77a5f6a-199d-446e-80f4-84a60856a1b8 tempest-ImagesTestJSON-1235141815 tempest-ImagesTestJSON-1235141815-project-member] Lock "53625c0d-203a-45f0-b899-5d2673413a53" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.845s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 907.801519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 7b157db86579476c856890a066be474e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.806124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg ea80382675a74de8a8bb89c3dc56077c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 907.807280] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b157db86579476c856890a066be474e [ 907.837360] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea80382675a74de8a8bb89c3dc56077c [ 908.023458] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.733s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 908.024125] env[62736]: ERROR nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Traceback (most recent call last): [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.driver.spawn(context, instance, image_meta, [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self._vmops.spawn(context, instance, image_meta, injected_files, [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] vm_ref = self.build_virtual_machine(instance, [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] vif_infos = vmwarevif.get_vif_info(self._session, [ 908.024125] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] for vif in network_info: [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return self._sync_wrapper(fn, *args, **kwargs) [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.wait() [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self[:] = self._gt.wait() [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return self._exit_event.wait() [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] current.throw(*self._exc) [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 908.024562] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] result = function(*args, **kwargs) [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] return func(*args, **kwargs) [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise e [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] nwinfo = self.network_api.allocate_for_instance( [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] created_port_ids = self._update_ports_for_instance( [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] with excutils.save_and_reraise_exception(): [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] self.force_reraise() [ 908.025017] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise self.value [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] updated_port = self._update_port( [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] _ensure_no_port_binding_failure(port) [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] raise exception.PortBindingFailed(port_id=port['id']) [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] nova.exception.PortBindingFailed: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. [ 908.025392] env[62736]: ERROR nova.compute.manager [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] [ 908.025392] env[62736]: DEBUG nova.compute.utils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 908.026058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 19.653s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 908.027961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg d7beedec69c5462aa0e650e39cc93c4d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.029165] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Build of instance f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90 was re-scheduled: Binding failed for port f5acee40-6e6c-4fea-a969-034bb88c660e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 908.029602] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 908.029823] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquiring lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.029961] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Acquired lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.030109] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 908.030482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg c5a915e15d7e46fab0100c2ad8435938 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.036851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5a915e15d7e46fab0100c2ad8435938 [ 908.074240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7beedec69c5462aa0e650e39cc93c4d [ 908.081432] env[62736]: DEBUG nova.compute.manager [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Received event network-changed-f4444a92-d000-41ce-be26-294346c5047b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 908.081611] env[62736]: DEBUG nova.compute.manager [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Refreshing instance network info cache due to event network-changed-f4444a92-d000-41ce-be26-294346c5047b. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 908.081816] env[62736]: DEBUG oslo_concurrency.lockutils [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] Acquiring lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.081949] env[62736]: DEBUG oslo_concurrency.lockutils [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] Acquired lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 908.082095] env[62736]: DEBUG nova.network.neutron [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Refreshing network info cache for port f4444a92-d000-41ce-be26-294346c5047b {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 908.082493] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] Expecting reply to msg 42aca68b25e749a980f3bcc638b05b8c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.088480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 42aca68b25e749a980f3bcc638b05b8c [ 908.222364] env[62736]: ERROR nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 908.222364] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 908.222364] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 908.222364] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 908.222364] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 908.222364] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 908.222364] env[62736]: ERROR nova.compute.manager raise self.value [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 908.222364] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 908.222364] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 908.222364] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 908.223024] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 908.223024] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 908.223024] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 908.223024] env[62736]: ERROR nova.compute.manager [ 908.223024] env[62736]: Traceback (most recent call last): [ 908.223024] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 908.223024] env[62736]: listener.cb(fileno) [ 908.223024] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 908.223024] env[62736]: result = function(*args, **kwargs) [ 908.223024] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 908.223024] env[62736]: return func(*args, **kwargs) [ 908.223024] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 908.223024] env[62736]: raise e [ 908.223024] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 908.223024] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 908.223024] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 908.223024] env[62736]: created_port_ids = self._update_ports_for_instance( [ 908.223024] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 908.223024] env[62736]: with excutils.save_and_reraise_exception(): [ 908.223024] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 908.223024] env[62736]: self.force_reraise() [ 908.223024] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 908.223024] env[62736]: raise self.value [ 908.223024] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 908.223024] env[62736]: updated_port = self._update_port( [ 908.223024] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 908.223024] env[62736]: _ensure_no_port_binding_failure(port) [ 908.223024] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 908.223024] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 908.223940] env[62736]: nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 908.223940] env[62736]: Removing descriptor: 16 [ 908.298252] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 908.300182] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg b7ce9789af56458caf8fc5781917cf10 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.309590] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 908.332229] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b7ce9789af56458caf8fc5781917cf10 [ 908.338062] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 908.338288] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 908.338454] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 908.338612] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 908.338753] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 908.338895] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 908.339094] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 908.339245] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 908.339410] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 908.339642] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 908.339810] env[62736]: DEBUG nova.virt.hardware [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 908.340978] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db79147f-524a-4570-8515-e512d665e3d7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.349259] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a21b7c8-b9c5-429c-801c-bf09b6caaa97 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.362955] env[62736]: ERROR nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Traceback (most recent call last): [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] yield resources [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.driver.spawn(context, instance, image_meta, [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] vm_ref = self.build_virtual_machine(instance, [ 908.362955] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] vif_infos = vmwarevif.get_vif_info(self._session, [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] for vif in network_info: [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return self._sync_wrapper(fn, *args, **kwargs) [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.wait() [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self[:] = self._gt.wait() [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return self._exit_event.wait() [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 908.363292] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] current.throw(*self._exc) [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] result = function(*args, **kwargs) [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return func(*args, **kwargs) [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise e [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] nwinfo = self.network_api.allocate_for_instance( [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] created_port_ids = self._update_ports_for_instance( [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] with excutils.save_and_reraise_exception(): [ 908.363628] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.force_reraise() [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise self.value [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] updated_port = self._update_port( [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] _ensure_no_port_binding_failure(port) [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise exception.PortBindingFailed(port_id=port['id']) [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 908.363965] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] [ 908.363965] env[62736]: INFO nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Terminating instance [ 908.365017] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 908.551693] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.633908] env[62736]: DEBUG nova.network.neutron [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.646663] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 908.647164] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 722de2781f834f018f3cea0f6a4f3274 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.655030] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 722de2781f834f018f3cea0f6a4f3274 [ 908.703665] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-836c9297-5efd-4b74-94c6-4fdfd62360b4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.711924] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9800c97-91b7-4de2-9198-6bf7ba4dbe0d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.747657] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b05c2c96-22d5-47ba-ae07-9b6c823e2a7a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.755116] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0db4d3e-8ed7-4401-bd99-31debc406102 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 908.768488] env[62736]: DEBUG nova.compute.provider_tree [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 908.768994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg fc835dbf3dbb48bab4f2339f8e48044e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.775931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc835dbf3dbb48bab4f2339f8e48044e [ 908.818053] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 908.855616] env[62736]: DEBUG nova.network.neutron [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 908.856498] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] Expecting reply to msg 7a1f443f70c446ea898527b63315fd12 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 908.868883] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a1f443f70c446ea898527b63315fd12 [ 909.150062] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Releasing lock "refresh_cache-f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.150309] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 909.150487] env[62736]: DEBUG nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 909.150653] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 909.170069] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 909.170687] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg ec14fde610994f84b3bf78ca7a6ee0f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.178166] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec14fde610994f84b3bf78ca7a6ee0f0 [ 909.271533] env[62736]: DEBUG nova.scheduler.client.report [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 909.273800] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg a5e6e62bad0f4d0ab277f14945072373 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.286725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5e6e62bad0f4d0ab277f14945072373 [ 909.362884] env[62736]: DEBUG oslo_concurrency.lockutils [req-e42e0b88-afa6-4a03-bc9e-a0af0316cfa7 req-87ccd81e-4bd2-488a-a49d-962c2253aaf2 service nova] Releasing lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 909.363382] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 909.363537] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 909.364041] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 4552ffb3dde54b05b161221eb18ee268 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.377247] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4552ffb3dde54b05b161221eb18ee268 [ 909.673315] env[62736]: DEBUG nova.network.neutron [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.673963] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 587cfa802a97422da81c3d91cd8204ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.684267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 587cfa802a97422da81c3d91cd8204ca [ 909.777169] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.751s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 909.777705] env[62736]: ERROR nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Traceback (most recent call last): [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.driver.spawn(context, instance, image_meta, [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self._vmops.spawn(context, instance, image_meta, injected_files, [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] vm_ref = self.build_virtual_machine(instance, [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] vif_infos = vmwarevif.get_vif_info(self._session, [ 909.777705] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] for vif in network_info: [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return self._sync_wrapper(fn, *args, **kwargs) [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.wait() [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self[:] = self._gt.wait() [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return self._exit_event.wait() [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] current.throw(*self._exc) [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 909.778062] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] result = function(*args, **kwargs) [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] return func(*args, **kwargs) [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise e [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] nwinfo = self.network_api.allocate_for_instance( [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] created_port_ids = self._update_ports_for_instance( [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] with excutils.save_and_reraise_exception(): [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] self.force_reraise() [ 909.778497] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise self.value [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] updated_port = self._update_port( [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] _ensure_no_port_binding_failure(port) [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] raise exception.PortBindingFailed(port_id=port['id']) [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] nova.exception.PortBindingFailed: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. [ 909.778877] env[62736]: ERROR nova.compute.manager [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] [ 909.778877] env[62736]: DEBUG nova.compute.utils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 909.779666] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 13.050s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 909.781442] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg c715ea57d86a4ca1a294d5add9b341f6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.782920] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Build of instance 570ee4e0-caeb-41f9-82d2-fa009f23581d was re-scheduled: Binding failed for port 57fb7c5e-9fe9-48df-ac1b-5a913e230491, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 909.783223] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 909.783286] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 909.783385] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 909.783538] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 909.783890] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 30db68d99b894bc4b232c38d9207529d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.791979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30db68d99b894bc4b232c38d9207529d [ 909.820760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c715ea57d86a4ca1a294d5add9b341f6 [ 909.885970] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 909.975635] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 909.976240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 997d1a1042ce4df1936b263a4ef13f10 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 909.984300] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 997d1a1042ce4df1936b263a4ef13f10 [ 910.108428] env[62736]: DEBUG nova.compute.manager [req-0a4672d9-d886-4b4b-8563-0ebdc4909ccd req-38174610-897b-4140-8aec-a625ae935957 service nova] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Received event network-vif-deleted-f4444a92-d000-41ce-be26-294346c5047b {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 910.176717] env[62736]: INFO nova.compute.manager [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] [instance: f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90] Took 1.03 seconds to deallocate network for instance. [ 910.178426] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 1c876a57f0904e3faf492aea0d4faaec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.217773] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c876a57f0904e3faf492aea0d4faaec [ 910.308927] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 910.395733] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 910.396252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 643bea288a7c4ac8839be40fc7974d79 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.407580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 643bea288a7c4ac8839be40fc7974d79 [ 910.429747] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac4b7146-ae14-4c64-9d41-38da43289f90 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.437509] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7d75262-655a-4946-9ff5-f37d23ae4185 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.466463] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-941be1cc-64ac-4ae7-a018-12f2de247258 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.474108] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-386197ca-14ba-49e9-9078-47d01e1e3801 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.478649] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 910.479014] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 910.479189] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 910.479482] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cdf9a01a-d855-42f9-b635-80394441b115 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.490102] env[62736]: DEBUG nova.compute.provider_tree [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 910.490536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 8489975f7e4f48de8aa9878ddb65aeba in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.495260] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf45cbee-fbb5-4c6e-b5f4-e7fd697d7633 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 910.505700] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8489975f7e4f48de8aa9878ddb65aeba [ 910.517203] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c1001bab-08b9-4bb9-b737-ba7fa5f929f0 could not be found. [ 910.517408] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 910.517574] env[62736]: INFO nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Took 0.04 seconds to destroy the instance on the hypervisor. [ 910.517798] env[62736]: DEBUG oslo.service.loopingcall [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 910.517999] env[62736]: DEBUG nova.compute.manager [-] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 910.518092] env[62736]: DEBUG nova.network.neutron [-] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 910.537575] env[62736]: DEBUG nova.network.neutron [-] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 910.537575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f578be00df334abdb2766ee6df0026a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.543870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f578be00df334abdb2766ee6df0026a3 [ 910.683234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg 709865b416a041e080d6555c35de444a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.715728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 709865b416a041e080d6555c35de444a [ 910.901804] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-570ee4e0-caeb-41f9-82d2-fa009f23581d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 910.902051] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 910.902231] env[62736]: DEBUG nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 910.902393] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 910.939869] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 910.940577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg df9dd350498943cfb75479ed420beb99 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 910.947705] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df9dd350498943cfb75479ed420beb99 [ 910.993521] env[62736]: DEBUG nova.scheduler.client.report [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 910.996040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 90e2080476bb4e7c9327948dae81704e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.008689] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90e2080476bb4e7c9327948dae81704e [ 911.038991] env[62736]: DEBUG nova.network.neutron [-] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 911.039480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg b400bcd2957d49d4868e12ec01968f87 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.047217] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b400bcd2957d49d4868e12ec01968f87 [ 911.208844] env[62736]: INFO nova.scheduler.client.report [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Deleted allocations for instance f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90 [ 911.240809] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Expecting reply to msg b2b505975b524f45bc6ddd1fb94c6ca3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.240809] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2b505975b524f45bc6ddd1fb94c6ca3 [ 911.444147] env[62736]: DEBUG nova.network.neutron [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 911.444357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 6474911ee3944fabb85ab178a0f75dfc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.452425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6474911ee3944fabb85ab178a0f75dfc [ 911.498453] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.719s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 911.499096] env[62736]: ERROR nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Traceback (most recent call last): [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.driver.spawn(context, instance, image_meta, [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self._vmops.spawn(context, instance, image_meta, injected_files, [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] vm_ref = self.build_virtual_machine(instance, [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] vif_infos = vmwarevif.get_vif_info(self._session, [ 911.499096] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] for vif in network_info: [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return self._sync_wrapper(fn, *args, **kwargs) [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.wait() [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self[:] = self._gt.wait() [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return self._exit_event.wait() [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] current.throw(*self._exc) [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 911.499596] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] result = function(*args, **kwargs) [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] return func(*args, **kwargs) [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise e [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] nwinfo = self.network_api.allocate_for_instance( [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] created_port_ids = self._update_ports_for_instance( [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] with excutils.save_and_reraise_exception(): [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] self.force_reraise() [ 911.499939] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise self.value [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] updated_port = self._update_port( [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] _ensure_no_port_binding_failure(port) [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] raise exception.PortBindingFailed(port_id=port['id']) [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] nova.exception.PortBindingFailed: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. [ 911.500316] env[62736]: ERROR nova.compute.manager [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] [ 911.500316] env[62736]: DEBUG nova.compute.utils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 911.501175] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.032s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 911.502530] env[62736]: INFO nova.compute.claims [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 911.504145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 40667854b28049e8886dbff8f4da0e97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.505143] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Build of instance 7d047648-facb-4d58-acf7-a99a3f7adf65 was re-scheduled: Binding failed for port 53a6658a-47e5-46fc-bdd1-b2efc0b47597, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 911.505551] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 911.505762] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquiring lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 911.505901] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Acquired lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 911.506215] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 911.506415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 78482e7134d0406f9d38fe5b43d05f68 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.512382] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78482e7134d0406f9d38fe5b43d05f68 [ 911.538010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 40667854b28049e8886dbff8f4da0e97 [ 911.544312] env[62736]: INFO nova.compute.manager [-] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Took 1.02 seconds to deallocate network for instance. [ 911.545030] env[62736]: DEBUG nova.compute.claims [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 911.545201] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 911.723046] env[62736]: DEBUG oslo_concurrency.lockutils [None req-353a033c-9052-4537-872c-291d68108f1a tempest-VolumesAdminNegativeTest-1053254376 tempest-VolumesAdminNegativeTest-1053254376-project-member] Lock "f7ed0a84-a3ca-46f8-8cfb-da2311c9cb90" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 88.756s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 911.723669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 990ea0e0e8c643e1933e242414d3f399 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 911.733275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 990ea0e0e8c643e1933e242414d3f399 [ 911.946612] env[62736]: INFO nova.compute.manager [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: 570ee4e0-caeb-41f9-82d2-fa009f23581d] Took 1.04 seconds to deallocate network for instance. [ 911.948523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 2a9a43f1b77b461eb6c05a908aed1f96 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.001582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2a9a43f1b77b461eb6c05a908aed1f96 [ 912.009345] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 6c6dcd27f5bc4054abee23898d84b59f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.020194] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c6dcd27f5bc4054abee23898d84b59f [ 912.030454] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 912.136940] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 912.136940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 1a62c4d4b78d40438e252de9575ab6fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.144716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a62c4d4b78d40438e252de9575ab6fe [ 912.228791] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 912.228791] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 64ad39f06dc94109bf37848cdd2fa1b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.265409] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64ad39f06dc94109bf37848cdd2fa1b7 [ 912.456761] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 8c618704f1124a9e8c03369902bca45e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.500647] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c618704f1124a9e8c03369902bca45e [ 912.638330] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Releasing lock "refresh_cache-7d047648-facb-4d58-acf7-a99a3f7adf65" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 912.638556] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 912.638730] env[62736]: DEBUG nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 912.638884] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 912.654769] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e403cc29-cffd-4a81-a2fe-5c1aa3fdf9a3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.657951] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 912.658508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 45bd2783bc7448cfaed10fc54458b170 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.665994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45bd2783bc7448cfaed10fc54458b170 [ 912.667104] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8451f484-0d5e-4b6a-a7e2-e0f127db2bb6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.697264] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ed96208-4bb6-456e-95c7-08811b2dbe9f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.704576] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61226e44-f330-4c40-bdc7-34e2a9a0f94a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 912.718833] env[62736]: DEBUG nova.compute.provider_tree [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 912.719333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 1814959ff35646ec810e8595c5d62d46 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 912.726534] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1814959ff35646ec810e8595c5d62d46 [ 912.744001] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 912.990846] env[62736]: INFO nova.scheduler.client.report [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Deleted allocations for instance 570ee4e0-caeb-41f9-82d2-fa009f23581d [ 913.001068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 5ed8a175cf9e43958ad1809acc5f4c74 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.012711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ed8a175cf9e43958ad1809acc5f4c74 [ 913.160599] env[62736]: DEBUG nova.network.neutron [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 913.161132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 9327b48539a94ce6ba82dfc4676e324b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.170832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9327b48539a94ce6ba82dfc4676e324b [ 913.222289] env[62736]: DEBUG nova.scheduler.client.report [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 913.224709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 311b6e15b93149d1b13299e4ef51110c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.235909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 311b6e15b93149d1b13299e4ef51110c [ 913.498826] env[62736]: DEBUG oslo_concurrency.lockutils [None req-ce360136-9a77-4eae-9882-377646ff5edd tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "570ee4e0-caeb-41f9-82d2-fa009f23581d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 90.183s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 913.663782] env[62736]: INFO nova.compute.manager [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] [instance: 7d047648-facb-4d58-acf7-a99a3f7adf65] Took 1.02 seconds to deallocate network for instance. [ 913.665679] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 27900cd293a842abb93dcce2ead8b3d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.708698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27900cd293a842abb93dcce2ead8b3d5 [ 913.727857] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.226s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 913.727951] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 913.729925] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 4be5d61f83584db9879102dd500feae5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.730941] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 12.673s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 913.732966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 0526eb025aea47ba8649976d79ad971f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 913.762780] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4be5d61f83584db9879102dd500feae5 [ 913.773407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0526eb025aea47ba8649976d79ad971f [ 914.170715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 25536dd2e9984a19b4d51e0ebbf9e6f5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 914.213768] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 25536dd2e9984a19b4d51e0ebbf9e6f5 [ 914.235802] env[62736]: DEBUG nova.compute.utils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 914.236405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 43c9fe63ae454a70b5f364bbe36278de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 914.243903] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 914.254910] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 43c9fe63ae454a70b5f364bbe36278de [ 914.420644] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f379161-990f-4be1-88c2-56b6161276ae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.427203] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb9b4fe1-a5fd-452c-9851-379c24a2f3e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.459639] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f69bc0c3-5dfd-4e08-905a-1bf931893e8c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.467519] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-779257da-3e10-4d4a-a130-86c572da44a6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 914.483691] env[62736]: DEBUG nova.compute.provider_tree [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 914.484425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg fb4f857544d746f2ae4107d78a65df16 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 914.491897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb4f857544d746f2ae4107d78a65df16 [ 914.695040] env[62736]: INFO nova.scheduler.client.report [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Deleted allocations for instance 7d047648-facb-4d58-acf7-a99a3f7adf65 [ 914.703090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Expecting reply to msg 181be9f6e9e44c6b88d2f7cf34d9a7ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 914.719264] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 181be9f6e9e44c6b88d2f7cf34d9a7ec [ 914.746050] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 914.748575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg f3371659611d45deabc9f3291cb52852 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 914.796236] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3371659611d45deabc9f3291cb52852 [ 914.987053] env[62736]: DEBUG nova.scheduler.client.report [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 914.989632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 629b35e080c24b7ab273eb8e3bbffe8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 915.001792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 629b35e080c24b7ab273eb8e3bbffe8b [ 915.205234] env[62736]: DEBUG oslo_concurrency.lockutils [None req-179b36cc-f462-46f0-80f6-f565d4e6c368 tempest-ServerRescueTestJSON-1048251529 tempest-ServerRescueTestJSON-1048251529-project-member] Lock "7d047648-facb-4d58-acf7-a99a3f7adf65" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 85.929s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 915.253140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 11b7a75c143a428ea24a014652b64561 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 915.292786] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11b7a75c143a428ea24a014652b64561 [ 915.493461] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.762s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 915.494140] env[62736]: ERROR nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Traceback (most recent call last): [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.driver.spawn(context, instance, image_meta, [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self._vmops.spawn(context, instance, image_meta, injected_files, [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] vm_ref = self.build_virtual_machine(instance, [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] vif_infos = vmwarevif.get_vif_info(self._session, [ 915.494140] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] for vif in network_info: [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return self._sync_wrapper(fn, *args, **kwargs) [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.wait() [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self[:] = self._gt.wait() [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return self._exit_event.wait() [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] current.throw(*self._exc) [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 915.494457] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] result = function(*args, **kwargs) [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] return func(*args, **kwargs) [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise e [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] nwinfo = self.network_api.allocate_for_instance( [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] created_port_ids = self._update_ports_for_instance( [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] with excutils.save_and_reraise_exception(): [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] self.force_reraise() [ 915.494776] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise self.value [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] updated_port = self._update_port( [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] _ensure_no_port_binding_failure(port) [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] raise exception.PortBindingFailed(port_id=port['id']) [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] nova.exception.PortBindingFailed: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. [ 915.495091] env[62736]: ERROR nova.compute.manager [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] [ 915.495091] env[62736]: DEBUG nova.compute.utils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 915.496538] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 13.026s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 915.498115] env[62736]: INFO nova.compute.claims [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 915.499721] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 1aa7d3502bbe4919ae0130675c0b11d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 915.501137] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Build of instance fa27cfe2-6f7f-4397-bcc5-147d6865f404 was re-scheduled: Binding failed for port 343f61a7-4e93-40f6-87ad-1e52b67a570e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 915.501267] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 915.501583] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquiring lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 915.501665] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Acquired lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 915.501890] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 915.502113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 3c39873c8d324042adc26405f6f18838 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 915.508717] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c39873c8d324042adc26405f6f18838 [ 915.538763] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aa7d3502bbe4919ae0130675c0b11d1 [ 915.756736] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 915.764320] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "d02ada4d-6801-4353-a664-779c3cd5f163" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 915.764511] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "d02ada4d-6801-4353-a664-779c3cd5f163" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 915.765002] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg f05f2d8d08db4c1a93c565003ca10e45 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 915.780091] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 915.780353] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 915.780527] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 915.780718] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 915.780870] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 915.781010] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 915.781207] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 915.781360] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 915.781594] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 915.781657] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 915.781813] env[62736]: DEBUG nova.virt.hardware [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 915.782985] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9cacd5cb-063d-4936-a67f-85a3503caf10 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.785889] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f05f2d8d08db4c1a93c565003ca10e45 [ 915.792406] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7946314-9339-4b0e-bb58-b62d526a3a26 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.807573] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 915.814855] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Creating folder: Project (53cc41eb847d485196f118421cb1505a). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 915.814996] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c01ba61f-3fb4-4126-aeaa-08a9a419cfb0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.827114] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Created folder: Project (53cc41eb847d485196f118421cb1505a) in parent group-v107913. [ 915.827401] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Creating folder: Instances. Parent ref: group-v107938. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 915.827797] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-58cc647c-170e-4c68-af7d-aea3459a573b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.837663] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Created folder: Instances in parent group-v107938. [ 915.837932] env[62736]: DEBUG oslo.service.loopingcall [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 915.838314] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 915.838314] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-7acd0837-6214-4a95-9612-b982bb7bcb6e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 915.854995] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 915.854995] env[62736]: value = "task-397768" [ 915.854995] env[62736]: _type = "Task" [ 915.854995] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 915.862890] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397768, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 916.005234] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 4895008ca59847e0bf8b8bd98db463bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 916.013945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4895008ca59847e0bf8b8bd98db463bf [ 916.025303] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 916.224143] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.224609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 3ccc6384df7b45709527757281e93af1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 916.233806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3ccc6384df7b45709527757281e93af1 [ 916.267294] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 916.269227] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 37a0cad4f94b454fb4fb1c728a9fb344 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 916.311413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 37a0cad4f94b454fb4fb1c728a9fb344 [ 916.384102] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397768, 'name': CreateVM_Task, 'duration_secs': 0.25005} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.384263] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 916.384708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.384857] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.385193] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 916.385457] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d4cf0da7-9cee-432c-a54b-96214c834e23 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.390237] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 916.390237] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52a0b87f-d265-7d63-e238-871027713ac6" [ 916.390237] env[62736]: _type = "Task" [ 916.390237] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 916.399740] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52a0b87f-d265-7d63-e238-871027713ac6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 916.652686] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c7a1521-ff02-42b4-aafd-35981605cfcb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.660168] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-594fd35b-2e06-4e32-807c-2a597ff78514 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.700993] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48f1e2ec-aa88-4297-b5c9-951248a50a6f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.709531] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-833e846a-5319-40a0-be5a-1265ddd7151d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.725366] env[62736]: DEBUG nova.compute.provider_tree [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 916.726094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 74ec5f4a305c47229b4e7618be2a7530 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 916.727424] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Releasing lock "refresh_cache-fa27cfe2-6f7f-4397-bcc5-147d6865f404" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 916.727694] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 916.727935] env[62736]: DEBUG nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 916.728418] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 916.733879] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74ec5f4a305c47229b4e7618be2a7530 [ 916.753838] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 916.754599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 332f127331464fcb8e16d4d1e05fd1c2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 916.762602] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 332f127331464fcb8e16d4d1e05fd1c2 [ 916.799815] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 916.901441] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52a0b87f-d265-7d63-e238-871027713ac6, 'name': SearchDatastore_Task, 'duration_secs': 0.010089} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.901773] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 916.902022] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 916.902259] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 916.902404] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 916.902577] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 916.902833] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e3c0fcef-2c8d-4eb3-99b2-932593e4b45e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.911419] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 916.911614] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 916.912437] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b42ce31e-1479-41ea-a691-dc332f8146c1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.917862] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 916.917862] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52b9b53a-e7f1-62b2-3078-5edc5d489f82" [ 916.917862] env[62736]: _type = "Task" [ 916.917862] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 916.934348] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52b9b53a-e7f1-62b2-3078-5edc5d489f82, 'name': SearchDatastore_Task, 'duration_secs': 0.007159} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 916.935555] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c0187a0c-8921-4adc-83bd-62d500f16691 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 916.942663] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 916.942663] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52636a05-2bfe-db2f-530e-bff0f62970ab" [ 916.942663] env[62736]: _type = "Task" [ 916.942663] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 916.954291] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52636a05-2bfe-db2f-530e-bff0f62970ab, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 917.232265] env[62736]: DEBUG nova.scheduler.client.report [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 917.234642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ef009b366c00438cbd152ef2aec503a5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 917.250438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef009b366c00438cbd152ef2aec503a5 [ 917.258637] env[62736]: DEBUG nova.network.neutron [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 917.259097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 933017c732d8460dba772c7f1f97e8ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 917.266886] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 933017c732d8460dba772c7f1f97e8ec [ 917.454452] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52636a05-2bfe-db2f-530e-bff0f62970ab, 'name': SearchDatastore_Task, 'duration_secs': 0.008832} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 917.454884] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 917.455261] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] e7d784c2-53f8-4da7-8c25-4bdf5362e94d/e7d784c2-53f8-4da7-8c25-4bdf5362e94d.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 917.455652] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1cb90c5e-5e38-445e-8889-0f182e682919 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.462898] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 917.462898] env[62736]: value = "task-397770" [ 917.462898] env[62736]: _type = "Task" [ 917.462898] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 917.472157] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397770, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 917.751149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.248s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 917.751149] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 917.751149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ce5d16651a7a43488d47fbccf036a11b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 917.751149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 10.599s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 917.751149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 9e3c9acebcb842ecad5e2de43c57e208 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 917.761366] env[62736]: INFO nova.compute.manager [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] [instance: fa27cfe2-6f7f-4397-bcc5-147d6865f404] Took 1.03 seconds to deallocate network for instance. [ 917.766349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 004a36249cf946e78d0968e39dea916a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 917.790270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9e3c9acebcb842ecad5e2de43c57e208 [ 917.799684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce5d16651a7a43488d47fbccf036a11b [ 917.824202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 004a36249cf946e78d0968e39dea916a [ 917.974297] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397770, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.453661} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 917.975115] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] e7d784c2-53f8-4da7-8c25-4bdf5362e94d/e7d784c2-53f8-4da7-8c25-4bdf5362e94d.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 917.975553] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 917.975943] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-f258326c-596a-4da9-9aa2-6eaa2c909f4b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 917.982270] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 917.982270] env[62736]: value = "task-397771" [ 917.982270] env[62736]: _type = "Task" [ 917.982270] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 917.996789] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397771, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.258923] env[62736]: DEBUG nova.compute.utils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 918.258923] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a6427bdfddff4d0aa453fb03a2dbe98b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.258923] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 918.258923] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 918.264391] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6427bdfddff4d0aa453fb03a2dbe98b [ 918.275286] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 52456e5fe5f64b8abaf4a708e828bc7e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.321388] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 52456e5fe5f64b8abaf4a708e828bc7e [ 918.336257] env[62736]: DEBUG nova.policy [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b785ade05de40c88f4bf3ef92e5a25f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '721d9e362be84a33aa7956177650d445', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 918.421088] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90cc1793-fbc5-43e3-8f46-44bf99241bb0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.428971] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1443f1e2-c708-4498-86f9-955c8a24b360 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.461352] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efce60ef-6c65-4422-8ac3-8d45697870c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.468725] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ac57e85-79c1-44cb-a6d7-1d34d98849fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.482024] env[62736]: DEBUG nova.compute.provider_tree [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 918.482400] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg ff97b83008f84181a64d7a6856ce2037 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.494309] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ff97b83008f84181a64d7a6856ce2037 [ 918.495765] env[62736]: DEBUG nova.scheduler.client.report [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 918.498608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 821982a485a14321ad26f5d2fb71af2a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.505250] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397771, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.065582} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 918.507859] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 918.507859] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba7b3a08-7f6b-431f-8860-90f467c06af3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.510756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 821982a485a14321ad26f5d2fb71af2a [ 918.527547] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Reconfiguring VM instance instance-00000053 to attach disk [datastore1] e7d784c2-53f8-4da7-8c25-4bdf5362e94d/e7d784c2-53f8-4da7-8c25-4bdf5362e94d.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 918.528107] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-2e5c95ad-24fd-4a49-8ddc-7ca665166621 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 918.548153] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 918.548153] env[62736]: value = "task-397773" [ 918.548153] env[62736]: _type = "Task" [ 918.548153] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 918.555705] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397773, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 918.653334] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Successfully created port: 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 918.759042] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 918.760914] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg c0142357d0084ccea7be26ed5a94c7c0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.807244] env[62736]: INFO nova.scheduler.client.report [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Deleted allocations for instance fa27cfe2-6f7f-4397-bcc5-147d6865f404 [ 918.819931] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Expecting reply to msg 3a44409f52314344b8e8bfe85e805326 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 918.855528] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a44409f52314344b8e8bfe85e805326 [ 918.858048] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0142357d0084ccea7be26ed5a94c7c0 [ 919.001116] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.253s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 919.001757] env[62736]: ERROR nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] Traceback (most recent call last): [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.driver.spawn(context, instance, image_meta, [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self._vmops.spawn(context, instance, image_meta, injected_files, [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] vm_ref = self.build_virtual_machine(instance, [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] vif_infos = vmwarevif.get_vif_info(self._session, [ 919.001757] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] for vif in network_info: [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self._sync_wrapper(fn, *args, **kwargs) [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.wait() [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self[:] = self._gt.wait() [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self._exit_event.wait() [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] result = hub.switch() [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 919.002116] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return self.greenlet.switch() [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] result = function(*args, **kwargs) [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] return func(*args, **kwargs) [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise e [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] nwinfo = self.network_api.allocate_for_instance( [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] created_port_ids = self._update_ports_for_instance( [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] with excutils.save_and_reraise_exception(): [ 919.002462] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] self.force_reraise() [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise self.value [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] updated_port = self._update_port( [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] _ensure_no_port_binding_failure(port) [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] raise exception.PortBindingFailed(port_id=port['id']) [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] nova.exception.PortBindingFailed: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. [ 919.002804] env[62736]: ERROR nova.compute.manager [instance: 768cf3b2-227f-463a-b192-79c50874cded] [ 919.003100] env[62736]: DEBUG nova.compute.utils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 919.004017] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.662s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 919.005807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e038869f8ef54f8293a3b36cd0886513 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.006873] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Build of instance 768cf3b2-227f-463a-b192-79c50874cded was re-scheduled: Binding failed for port 68d97bd5-2b32-4ce1-bb1d-5d90628a1951, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 919.007392] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 919.007535] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquiring lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.007690] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Acquired lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.007849] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 919.008224] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 2e544dbd34424f0c9f7db0c9bd53e258 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.014665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2e544dbd34424f0c9f7db0c9bd53e258 [ 919.036584] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e038869f8ef54f8293a3b36cd0886513 [ 919.060046] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397773, 'name': ReconfigVM_Task, 'duration_secs': 0.306079} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.060428] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Reconfigured VM instance instance-00000053 to attach disk [datastore1] e7d784c2-53f8-4da7-8c25-4bdf5362e94d/e7d784c2-53f8-4da7-8c25-4bdf5362e94d.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 919.061373] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-99484eee-dfb2-45da-8bf8-49c559d506bb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.067695] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 919.067695] env[62736]: value = "task-397774" [ 919.067695] env[62736]: _type = "Task" [ 919.067695] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.078412] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397774, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.282214] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ec6a1395899f494a84456073ec155113 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.321774] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c6af30f2-763c-48ee-8cbc-70cd6ebca490 tempest-AttachInterfacesTestJSON-221845540 tempest-AttachInterfacesTestJSON-221845540-project-member] Lock "fa27cfe2-6f7f-4397-bcc5-147d6865f404" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 88.873s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 919.329849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ec6a1395899f494a84456073ec155113 [ 919.529290] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 919.581446] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397774, 'name': Rename_Task, 'duration_secs': 0.198982} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 919.583760] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 919.584169] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-14c3d768-227e-4881-90af-89f11acd6180 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.596941] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 919.596941] env[62736]: value = "task-397775" [ 919.596941] env[62736]: _type = "Task" [ 919.596941] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 919.610483] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397775, 'name': PowerOnVM_Task} progress is 33%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 919.626017] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 919.626554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 9f68d4c53d8b40b190d2a1e8005e89e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.646040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9f68d4c53d8b40b190d2a1e8005e89e5 [ 919.677353] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f37547ab-ed81-40b8-9c49-18d2559dab53 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.687359] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dff5a173-3c87-4bdb-ba00-5ca59ef30f5f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.720440] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51f7894d-8ae1-4d87-862e-c737fe30fdea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.724062] env[62736]: DEBUG nova.compute.manager [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Received event network-changed-3a7327d4-0399-4b0e-ba0d-a4cab0550f4f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 919.724250] env[62736]: DEBUG nova.compute.manager [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Refreshing instance network info cache due to event network-changed-3a7327d4-0399-4b0e-ba0d-a4cab0550f4f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 919.724461] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] Acquiring lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 919.724594] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] Acquired lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 919.724740] env[62736]: DEBUG nova.network.neutron [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Refreshing network info cache for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 919.725153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] Expecting reply to msg 3a80c4d6e62944909dcda83f420a5622 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.739189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3a80c4d6e62944909dcda83f420a5622 [ 919.740691] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1e60123-c358-471c-8fb6-ed68755d17fd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.755599] env[62736]: DEBUG nova.compute.provider_tree [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 919.756238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 64df15ab8bf646b89eb4e6f981dd1838 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 919.763902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64df15ab8bf646b89eb4e6f981dd1838 [ 919.785020] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 919.795264] env[62736]: ERROR nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 919.795264] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.795264] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 919.795264] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 919.795264] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.795264] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.795264] env[62736]: ERROR nova.compute.manager raise self.value [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 919.795264] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 919.795264] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.795264] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 919.795717] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.795717] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 919.795717] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 919.795717] env[62736]: ERROR nova.compute.manager [ 919.795717] env[62736]: Traceback (most recent call last): [ 919.795717] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 919.795717] env[62736]: listener.cb(fileno) [ 919.795717] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.795717] env[62736]: result = function(*args, **kwargs) [ 919.795717] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 919.795717] env[62736]: return func(*args, **kwargs) [ 919.795717] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.795717] env[62736]: raise e [ 919.795717] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.795717] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 919.795717] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 919.795717] env[62736]: created_port_ids = self._update_ports_for_instance( [ 919.795717] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 919.795717] env[62736]: with excutils.save_and_reraise_exception(): [ 919.795717] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.795717] env[62736]: self.force_reraise() [ 919.795717] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.795717] env[62736]: raise self.value [ 919.795717] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 919.795717] env[62736]: updated_port = self._update_port( [ 919.795717] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.795717] env[62736]: _ensure_no_port_binding_failure(port) [ 919.795717] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.795717] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 919.796491] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 919.796491] env[62736]: Removing descriptor: 17 [ 919.812449] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 919.812856] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 919.812856] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 919.813005] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 919.813291] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 919.813442] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 919.813649] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 919.813792] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 919.813951] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 919.814106] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 919.815062] env[62736]: DEBUG nova.virt.hardware [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 919.816112] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-163436c3-83e7-4934-a787-79d9130f0bc1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.827851] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44f70b8f-bd92-48ec-9414-4aa4917e4127 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 919.849067] env[62736]: ERROR nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Traceback (most recent call last): [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] yield resources [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.driver.spawn(context, instance, image_meta, [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] vm_ref = self.build_virtual_machine(instance, [ 919.849067] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] vif_infos = vmwarevif.get_vif_info(self._session, [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] for vif in network_info: [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return self._sync_wrapper(fn, *args, **kwargs) [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.wait() [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self[:] = self._gt.wait() [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return self._exit_event.wait() [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 919.849493] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] current.throw(*self._exc) [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] result = function(*args, **kwargs) [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return func(*args, **kwargs) [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise e [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] nwinfo = self.network_api.allocate_for_instance( [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] created_port_ids = self._update_ports_for_instance( [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] with excutils.save_and_reraise_exception(): [ 919.849872] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.force_reraise() [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise self.value [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] updated_port = self._update_port( [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] _ensure_no_port_binding_failure(port) [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise exception.PortBindingFailed(port_id=port['id']) [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 919.850238] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] [ 919.850238] env[62736]: INFO nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Terminating instance [ 919.851499] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 920.109651] env[62736]: DEBUG oslo_vmware.api [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397775, 'name': PowerOnVM_Task, 'duration_secs': 0.478835} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 920.109915] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 920.110105] env[62736]: INFO nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Took 4.35 seconds to spawn the instance on the hypervisor. [ 920.110272] env[62736]: DEBUG nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 920.111183] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89dc06dc-92c2-4315-9307-40c3e6ca294a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 920.118707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 331c0e99560e486ab762df24aab26942 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.146155] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Releasing lock "refresh_cache-768cf3b2-227f-463a-b192-79c50874cded" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.146409] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 920.146653] env[62736]: DEBUG nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 920.146841] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 920.153910] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 331c0e99560e486ab762df24aab26942 [ 920.169154] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 920.169742] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 10c276e1a88845fdaf35c3c8be7c273e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.176119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 10c276e1a88845fdaf35c3c8be7c273e [ 920.252599] env[62736]: DEBUG nova.network.neutron [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 920.258538] env[62736]: DEBUG nova.scheduler.client.report [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 920.260900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 487a00ed34ed49079ca92e9f8d61781a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.272882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 487a00ed34ed49079ca92e9f8d61781a [ 920.363366] env[62736]: DEBUG nova.network.neutron [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.363900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] Expecting reply to msg af7189e5b8e94b99a9c2e3f0e14c7aa1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.379640] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af7189e5b8e94b99a9c2e3f0e14c7aa1 [ 920.480662] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquiring lock "ca7fc450-351a-407f-96c0-21a39d037e54" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.480812] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "ca7fc450-351a-407f-96c0-21a39d037e54" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.481270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 441f2ba523bf4ed08f425e70e8d6c8b8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.493446] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 441f2ba523bf4ed08f425e70e8d6c8b8 [ 920.628927] env[62736]: INFO nova.compute.manager [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Took 22.18 seconds to build instance. [ 920.629470] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 516c277847894e2cbb337526587abef3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.640665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 516c277847894e2cbb337526587abef3 [ 920.673927] env[62736]: DEBUG nova.network.neutron [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.674600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 952b8643ca124757b0d42f39275802f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.687704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 952b8643ca124757b0d42f39275802f9 [ 920.701068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 61477ee1e87a4fb9a3061a3d4e59eef7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.709471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61477ee1e87a4fb9a3061a3d4e59eef7 [ 920.763609] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.759s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.764494] env[62736]: ERROR nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Traceback (most recent call last): [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.driver.spawn(context, instance, image_meta, [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self._vmops.spawn(context, instance, image_meta, injected_files, [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] vm_ref = self.build_virtual_machine(instance, [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] vif_infos = vmwarevif.get_vif_info(self._session, [ 920.764494] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] for vif in network_info: [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return self._sync_wrapper(fn, *args, **kwargs) [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.wait() [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self[:] = self._gt.wait() [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return self._exit_event.wait() [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] current.throw(*self._exc) [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 920.764829] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] result = function(*args, **kwargs) [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] return func(*args, **kwargs) [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise e [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] nwinfo = self.network_api.allocate_for_instance( [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] created_port_ids = self._update_ports_for_instance( [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] with excutils.save_and_reraise_exception(): [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] self.force_reraise() [ 920.765150] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise self.value [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] updated_port = self._update_port( [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] _ensure_no_port_binding_failure(port) [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] raise exception.PortBindingFailed(port_id=port['id']) [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] nova.exception.PortBindingFailed: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. [ 920.765792] env[62736]: ERROR nova.compute.manager [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] [ 920.767021] env[62736]: DEBUG nova.compute.utils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 920.769196] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.951s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.770898] env[62736]: INFO nova.compute.claims [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 920.773267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 7964f5c12432404382c206a17833610e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.775950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 6b27e810b4384c74a7fa38ede1977973 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.782736] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Build of instance 1f1dd01d-006d-4569-8684-e1eb913ba5d4 was re-scheduled: Binding failed for port a06f0a41-bdaf-4090-bfe5-9dc6efaf2200, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 920.783663] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 920.784854] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 920.785171] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 920.785471] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 920.785984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a14e6737b67f47f383e414dc270bfd54 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.788490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b27e810b4384c74a7fa38ede1977973 [ 920.800634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a14e6737b67f47f383e414dc270bfd54 [ 920.812965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7964f5c12432404382c206a17833610e [ 920.868027] env[62736]: DEBUG oslo_concurrency.lockutils [req-0b56516c-befd-4750-b99c-79c8cb86682e req-abd34a9b-0fe2-498d-b7f8-ef1fa74bdbc6 service nova] Releasing lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 920.868027] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 920.868027] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 920.868210] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg d8ef3cae48a74728938c625a581e0e07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 920.875677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8ef3cae48a74728938c625a581e0e07 [ 920.983833] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 920.986178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 1208efd7e92048f9aa8df5b06cf8b293 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.024699] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1208efd7e92048f9aa8df5b06cf8b293 [ 921.131754] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f2402f43-648c-4532-8014-e754d1e35a23 tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 55.392s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.178066] env[62736]: INFO nova.compute.manager [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] [instance: 768cf3b2-227f-463a-b192-79c50874cded] Took 1.03 seconds to deallocate network for instance. [ 921.179824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 18f66ced2d784af0801d50e9b0534a4b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.201892] env[62736]: DEBUG nova.compute.manager [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 921.202824] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ebdb333b-1935-4db4-8d68-246cf02e1dac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.211043] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 36ce2a86187944b78f6748710c09a206 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.222016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18f66ced2d784af0801d50e9b0534a4b [ 921.232770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36ce2a86187944b78f6748710c09a206 [ 921.283111] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.283370] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.283579] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.284213] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.001s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 921.284400] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.286443] env[62736]: INFO nova.compute.manager [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Terminating instance [ 921.288057] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "refresh_cache-e7d784c2-53f8-4da7-8c25-4bdf5362e94d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 921.288228] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquired lock "refresh_cache-e7d784c2-53f8-4da7-8c25-4bdf5362e94d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 921.288392] env[62736]: DEBUG nova.network.neutron [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 921.288781] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg cab6187fc3364b1e838b27f6702762ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.290727] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg f08304221dba42a3aeca90550aea5abd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.295532] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cab6187fc3364b1e838b27f6702762ee [ 921.304612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f08304221dba42a3aeca90550aea5abd [ 921.311409] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.392928] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.393409] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 16b4bff2c2d945ad9358ec44ac28f9ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.400193] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.402364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16b4bff2c2d945ad9358ec44ac28f9ae [ 921.509440] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 921.551856] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.552402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg d8f85ec9ce4b4afab44cc853302c2f66 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.561195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d8f85ec9ce4b4afab44cc853302c2f66 [ 921.687690] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 311e1fde077c4d8aa28ae6a0fb98df9c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.715560] env[62736]: INFO nova.compute.manager [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] instance snapshotting [ 921.715560] env[62736]: DEBUG nova.objects.instance [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lazy-loading 'flavor' on Instance uuid e7d784c2-53f8-4da7-8c25-4bdf5362e94d {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 921.715560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 8dc2945833e44b17bc74db4d425da196 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.724608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 311e1fde077c4d8aa28ae6a0fb98df9c [ 921.741355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8dc2945833e44b17bc74db4d425da196 [ 921.744719] env[62736]: DEBUG nova.compute.manager [req-3bcc43e9-e560-4bfe-aa47-0a09ad04dbfe req-6373a828-6de5-4eea-a398-5a6db5c75ed8 service nova] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Received event network-vif-deleted-3a7327d4-0399-4b0e-ba0d-a4cab0550f4f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 921.826576] env[62736]: DEBUG nova.network.neutron [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.895258] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-1f1dd01d-006d-4569-8684-e1eb913ba5d4" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 921.895507] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 921.895743] env[62736]: DEBUG nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 921.895867] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 921.898380] env[62736]: DEBUG nova.network.neutron [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 921.898859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 5e6ce8fe8f2d46c39c8b9db2e451f6d3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.907428] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e6ce8fe8f2d46c39c8b9db2e451f6d3 [ 921.943062] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e646feb8-53de-43ad-8892-7c215c69fd83 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.950947] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 921.951525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 788311b1204d4f37b874e61b39bba0be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 921.955242] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4969bc7-b49d-4bfe-ba50-2e7a5852b9eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 921.989852] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 788311b1204d4f37b874e61b39bba0be [ 921.991097] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-366b7269-fbb9-4f4c-8aba-6167be4c48a8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.002801] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93c94de7-ab13-4262-babe-24bc2747d5e9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.018038] env[62736]: DEBUG nova.compute.provider_tree [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 922.018569] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 4b5698278da44787bc097966c70dd4c6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.028821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b5698278da44787bc097966c70dd4c6 [ 922.054790] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.055141] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 922.055301] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 922.055619] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e1b7abec-570a-4606-932b-9f19836183e2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.064426] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8de689ad-acfc-4ba5-b27d-52d510de85d4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.087751] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5 could not be found. [ 922.087979] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 922.088184] env[62736]: INFO nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 922.088538] env[62736]: DEBUG oslo.service.loopingcall [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 922.088727] env[62736]: DEBUG nova.compute.manager [-] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 922.088851] env[62736]: DEBUG nova.network.neutron [-] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 922.107114] env[62736]: DEBUG nova.network.neutron [-] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 922.107750] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c4491c3822d443b49259300990f01efe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.114605] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4491c3822d443b49259300990f01efe [ 922.209347] env[62736]: INFO nova.scheduler.client.report [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Deleted allocations for instance 768cf3b2-227f-463a-b192-79c50874cded [ 922.216333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Expecting reply to msg 8189ac49fa424b4ead9610c8c0e925f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.220727] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7f6cb38-35c7-44fa-84ba-bec191d6560b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.237481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8189ac49fa424b4ead9610c8c0e925f1 [ 922.238525] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78ac9d34-df83-42e6-9ac3-daebe7d8d45c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.248184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg a27fed5dd1904090b358ced5ee1ea81e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.285419] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a27fed5dd1904090b358ced5ee1ea81e [ 922.402795] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Releasing lock "refresh_cache-e7d784c2-53f8-4da7-8c25-4bdf5362e94d" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 922.403234] env[62736]: DEBUG nova.compute.manager [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 922.403420] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 922.404381] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb095893-37e8-4a2d-8219-249c55cf3cb8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.413888] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 922.414127] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-7661798d-028d-45c0-af10-18051b9edb59 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.419989] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 922.419989] env[62736]: value = "task-397777" [ 922.419989] env[62736]: _type = "Task" [ 922.419989] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 922.427635] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397777, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 922.457287] env[62736]: DEBUG nova.network.neutron [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.458028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 6b0bd01cc9e040cd9095b2c613db73e8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.466026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b0bd01cc9e040cd9095b2c613db73e8 [ 922.521251] env[62736]: DEBUG nova.scheduler.client.report [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 922.523715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 6f7562b3cb244313971992444c48f598 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.538215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f7562b3cb244313971992444c48f598 [ 922.610366] env[62736]: DEBUG nova.network.neutron [-] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 922.610912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg abf9749d2264486e80272af5aef6848f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.618601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abf9749d2264486e80272af5aef6848f [ 922.720481] env[62736]: DEBUG oslo_concurrency.lockutils [None req-adf97f78-5ff9-49ac-8a0e-a7457993d4b3 tempest-ServerGroupTestJSON-28427255 tempest-ServerGroupTestJSON-28427255-project-member] Lock "768cf3b2-227f-463a-b192-79c50874cded" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 65.984s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.750864] env[62736]: DEBUG nova.compute.manager [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Instance disappeared during snapshot {{(pid=62736) _snapshot_instance /opt/stack/nova/nova/compute/manager.py:4494}} [ 922.904728] env[62736]: DEBUG nova.compute.manager [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Found 0 images (rotation: 2) {{(pid=62736) _rotate_backups /opt/stack/nova/nova/compute/manager.py:4554}} [ 922.905298] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1ded6fb3-c8ae-4495-9d9b-ede49de1df8b tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg db2d320ffa2f4c02b8c6428397366c1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.921226] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db2d320ffa2f4c02b8c6428397366c1e [ 922.931435] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397777, 'name': PowerOffVM_Task, 'duration_secs': 0.105431} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 922.931777] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 922.931941] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 922.932222] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-89ebf159-3958-4581-9346-71268ec423e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.954350] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 922.954595] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 922.954773] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Deleting the datastore file [datastore1] e7d784c2-53f8-4da7-8c25-4bdf5362e94d {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 922.955163] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-633c994f-b45f-471e-8945-1aa48d48f52d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 922.960325] env[62736]: INFO nova.compute.manager [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 1f1dd01d-006d-4569-8684-e1eb913ba5d4] Took 1.06 seconds to deallocate network for instance. [ 922.962093] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg de5ddd60ad0144719d408943167fb889 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 922.964966] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for the task: (returnval){ [ 922.964966] env[62736]: value = "task-397780" [ 922.964966] env[62736]: _type = "Task" [ 922.964966] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 922.973827] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397780, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 923.007476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de5ddd60ad0144719d408943167fb889 [ 923.026876] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.257s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 923.027409] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 923.029869] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg f37415b0d7c54b9583763b469f72c4a2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.030800] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 11.485s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 923.032625] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 3456097a6c3143e3aea823443f68086b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.064657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f37415b0d7c54b9583763b469f72c4a2 [ 923.071337] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3456097a6c3143e3aea823443f68086b [ 923.113286] env[62736]: INFO nova.compute.manager [-] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Took 1.02 seconds to deallocate network for instance. [ 923.115639] env[62736]: DEBUG nova.compute.claims [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 923.115803] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 923.468661] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 94c0ca67ff1845af9f58971cd0fbcc7b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.478713] env[62736]: DEBUG oslo_vmware.api [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Task: {'id': task-397780, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.143568} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 923.478972] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 923.479186] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 923.479524] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 923.479605] env[62736]: INFO nova.compute.manager [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Took 1.08 seconds to destroy the instance on the hypervisor. [ 923.479818] env[62736]: DEBUG oslo.service.loopingcall [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 923.480010] env[62736]: DEBUG nova.compute.manager [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 923.480106] env[62736]: DEBUG nova.network.neutron [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 923.494697] env[62736]: DEBUG nova.network.neutron [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 923.495252] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5bfc8eda05b6419fac85b0a9db2a36c2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.501965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 94c0ca67ff1845af9f58971cd0fbcc7b [ 923.502490] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bfc8eda05b6419fac85b0a9db2a36c2 [ 923.535378] env[62736]: DEBUG nova.compute.utils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 923.535976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg c3232746e75548b0936195b9da1a65d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.540031] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 923.540031] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 923.546028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3232746e75548b0936195b9da1a65d7 [ 923.594192] env[62736]: DEBUG nova.policy [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9825cb6e55f8419299e0b4f909b722d6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e049414f9871493398a8e5a07c3ef938', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 923.651165] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f1cab36-158a-46f4-a3ea-2fee1ca443fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.658711] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfeb40c0-331d-4559-82c1-534f5c72f318 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.689592] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-103b71fd-090e-415b-a472-8484f25faf85 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.697736] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c84a41c-9efe-436b-bd75-4a44305ce5ef {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 923.712173] env[62736]: DEBUG nova.compute.provider_tree [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 923.712665] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2972478c4a444ec7a181a7f186384a0b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 923.720999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2972478c4a444ec7a181a7f186384a0b [ 923.885675] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Successfully created port: c6993fc6-8b41-4675-8af3-4c43f6501e88 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 923.997463] env[62736]: INFO nova.scheduler.client.report [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance 1f1dd01d-006d-4569-8684-e1eb913ba5d4 [ 924.003427] env[62736]: DEBUG nova.network.neutron [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 924.003769] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7006b4bfc971425c8d881a8e9d361ed5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.005529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 0b619c474c474b4eb7c17a91faa95152 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.012831] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7006b4bfc971425c8d881a8e9d361ed5 [ 924.030896] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b619c474c474b4eb7c17a91faa95152 [ 924.042218] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 924.044246] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg de7a56da5a984e718b5b4580962104ad in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.097733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de7a56da5a984e718b5b4580962104ad [ 924.216123] env[62736]: DEBUG nova.scheduler.client.report [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 924.218010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 082d6e4b2b0145388117d94b0e43d103 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.233562] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 082d6e4b2b0145388117d94b0e43d103 [ 924.506850] env[62736]: INFO nova.compute.manager [-] [instance: e7d784c2-53f8-4da7-8c25-4bdf5362e94d] Took 1.03 seconds to deallocate network for instance. [ 924.507334] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7d0de56-06ae-4508-808f-7a924de4bbc9 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "1f1dd01d-006d-4569-8684-e1eb913ba5d4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.367s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.513752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 31d75da10a634b14ba4f17d88eb320cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.545760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31d75da10a634b14ba4f17d88eb320cf [ 924.549098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg fc94b944aff74eca85a7b49932d934fb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.583311] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc94b944aff74eca85a7b49932d934fb [ 924.721050] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.690s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 924.721678] env[62736]: ERROR nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Traceback (most recent call last): [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.driver.spawn(context, instance, image_meta, [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] vm_ref = self.build_virtual_machine(instance, [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] vif_infos = vmwarevif.get_vif_info(self._session, [ 924.721678] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] for vif in network_info: [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return self._sync_wrapper(fn, *args, **kwargs) [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.wait() [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self[:] = self._gt.wait() [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return self._exit_event.wait() [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] current.throw(*self._exc) [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 924.722100] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] result = function(*args, **kwargs) [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] return func(*args, **kwargs) [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise e [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] nwinfo = self.network_api.allocate_for_instance( [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] created_port_ids = self._update_ports_for_instance( [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] with excutils.save_and_reraise_exception(): [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] self.force_reraise() [ 924.722491] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise self.value [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] updated_port = self._update_port( [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] _ensure_no_port_binding_failure(port) [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] raise exception.PortBindingFailed(port_id=port['id']) [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] nova.exception.PortBindingFailed: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. [ 924.722922] env[62736]: ERROR nova.compute.manager [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] [ 924.722922] env[62736]: DEBUG nova.compute.utils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 924.723585] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 11.980s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 924.725319] env[62736]: INFO nova.compute.claims [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 924.726832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 757342d4f0cd4e79a39cfbff769662fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.729248] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Build of instance c1001bab-08b9-4bb9-b737-ba7fa5f929f0 was re-scheduled: Binding failed for port f4444a92-d000-41ce-be26-294346c5047b, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 924.729731] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 924.729958] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 924.730091] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 924.730240] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 924.730688] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg be3b05f6aff94a5f824ffd30e7f122d8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 924.744398] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg be3b05f6aff94a5f824ffd30e7f122d8 [ 924.777413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 757342d4f0cd4e79a39cfbff769662fc [ 925.016421] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 925.052642] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 925.077521] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 925.077710] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 925.077861] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 925.078118] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 925.078264] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 925.078467] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 925.078745] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 925.078951] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 925.079452] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 925.079452] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 925.079677] env[62736]: DEBUG nova.virt.hardware [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 925.080633] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bbdc436-1311-4d04-ba74-491f90b2333c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.089033] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9019d734-4105-4e4b-a3b1-bba5d6d5fb3e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.233621] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg f6bcd03d5e1a4a6092818055ccfb24a9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.242764] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6bcd03d5e1a4a6092818055ccfb24a9 [ 925.260504] env[62736]: DEBUG nova.compute.manager [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Received event network-changed-c6993fc6-8b41-4675-8af3-4c43f6501e88 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 925.260685] env[62736]: DEBUG nova.compute.manager [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Refreshing instance network info cache due to event network-changed-c6993fc6-8b41-4675-8af3-4c43f6501e88. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 925.260981] env[62736]: DEBUG oslo_concurrency.lockutils [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] Acquiring lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 925.261086] env[62736]: DEBUG oslo_concurrency.lockutils [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] Acquired lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 925.261301] env[62736]: DEBUG nova.network.neutron [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Refreshing network info cache for port c6993fc6-8b41-4675-8af3-4c43f6501e88 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 925.262115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] Expecting reply to msg 22e8ecf58718414bbe1cf9a145cc98ac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.268249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22e8ecf58718414bbe1cf9a145cc98ac [ 925.274608] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 925.297442] env[62736]: DEBUG nova.network.neutron [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 925.398182] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 925.398698] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 36132509d92b4c2380a23704ce6ba9af in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.407810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36132509d92b4c2380a23704ce6ba9af [ 925.676331] env[62736]: DEBUG nova.network.neutron [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 925.677647] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] Expecting reply to msg 8a071b6c23f84a7c9b219322741094bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.681108] env[62736]: ERROR nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 925.681108] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.681108] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 925.681108] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 925.681108] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.681108] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.681108] env[62736]: ERROR nova.compute.manager raise self.value [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 925.681108] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 925.681108] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.681108] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 925.681667] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.681667] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 925.681667] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 925.681667] env[62736]: ERROR nova.compute.manager [ 925.681667] env[62736]: Traceback (most recent call last): [ 925.681667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 925.681667] env[62736]: listener.cb(fileno) [ 925.681667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 925.681667] env[62736]: result = function(*args, **kwargs) [ 925.681667] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 925.681667] env[62736]: return func(*args, **kwargs) [ 925.681667] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 925.681667] env[62736]: raise e [ 925.681667] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.681667] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 925.681667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 925.681667] env[62736]: created_port_ids = self._update_ports_for_instance( [ 925.681667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 925.681667] env[62736]: with excutils.save_and_reraise_exception(): [ 925.681667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.681667] env[62736]: self.force_reraise() [ 925.681667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.681667] env[62736]: raise self.value [ 925.681667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 925.681667] env[62736]: updated_port = self._update_port( [ 925.681667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.681667] env[62736]: _ensure_no_port_binding_failure(port) [ 925.681667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.681667] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 925.682666] env[62736]: nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 925.682666] env[62736]: Removing descriptor: 17 [ 925.682666] env[62736]: ERROR nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] Traceback (most recent call last): [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] yield resources [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.driver.spawn(context, instance, image_meta, [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 925.682666] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] vm_ref = self.build_virtual_machine(instance, [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] vif_infos = vmwarevif.get_vif_info(self._session, [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] for vif in network_info: [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self._sync_wrapper(fn, *args, **kwargs) [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.wait() [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self[:] = self._gt.wait() [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self._exit_event.wait() [ 925.683148] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] result = hub.switch() [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self.greenlet.switch() [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] result = function(*args, **kwargs) [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return func(*args, **kwargs) [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise e [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] nwinfo = self.network_api.allocate_for_instance( [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 925.683569] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] created_port_ids = self._update_ports_for_instance( [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] with excutils.save_and_reraise_exception(): [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.force_reraise() [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise self.value [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] updated_port = self._update_port( [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] _ensure_no_port_binding_failure(port) [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 925.683986] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise exception.PortBindingFailed(port_id=port['id']) [ 925.684507] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 925.684507] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] [ 925.684507] env[62736]: INFO nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Terminating instance [ 925.685439] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 925.691460] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8a071b6c23f84a7c9b219322741094bb [ 925.852503] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6bb422d7-21e0-41a7-af75-1a8f25a6b7eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.860569] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd5ddfd6-4f31-4669-bd78-f73acc001b24 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.891334] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-439b8ab9-a77d-41f3-a899-476d26fa72bd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.903471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-c1001bab-08b9-4bb9-b737-ba7fa5f929f0" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 925.903718] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 925.903895] env[62736]: DEBUG nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 925.904071] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 925.910925] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-081943da-e8f9-4511-8497-c8126847a468 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 925.927262] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 925.929156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1d4bff21059e4cde91a332e419937edb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.929156] env[62736]: DEBUG nova.compute.provider_tree [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 925.929745] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg c97cb15054be4c26bfa9e92ae443173c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 925.943917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d4bff21059e4cde91a332e419937edb [ 925.943917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c97cb15054be4c26bfa9e92ae443173c [ 926.179975] env[62736]: DEBUG oslo_concurrency.lockutils [req-1a0cca40-ead8-407f-9e37-72c159ddf284 req-39ebf64b-782b-4ea1-9a6b-2d3c842ae53c service nova] Releasing lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 926.180452] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquired lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 926.180639] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 926.181204] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg f8050665864d4138be3927ad51d5f417 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.187673] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8050665864d4138be3927ad51d5f417 [ 926.435328] env[62736]: DEBUG nova.network.neutron [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 926.435328] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2b52a3944de7468ba9609d2197221131 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.435328] env[62736]: DEBUG nova.scheduler.client.report [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 926.436170] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 63af46ee6db5472ea6b794769f0161fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.443684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b52a3944de7468ba9609d2197221131 [ 926.459160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63af46ee6db5472ea6b794769f0161fa [ 926.594508] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "98ce815b-3e40-424f-bad6-a1b6a49e0a74" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 926.594722] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "98ce815b-3e40-424f-bad6-a1b6a49e0a74" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 926.595168] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d60dcf20ab514f52b9e014d347e6e3d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.606191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d60dcf20ab514f52b9e014d347e6e3d6 [ 926.698354] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 926.779019] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 926.779608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg faedf64801ba4e6b8016a55aa1835244 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.788147] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg faedf64801ba4e6b8016a55aa1835244 [ 926.937792] env[62736]: INFO nova.compute.manager [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: c1001bab-08b9-4bb9-b737-ba7fa5f929f0] Took 1.03 seconds to deallocate network for instance. [ 926.939530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a8d3c11c04d94742b24a7e138779cba8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.940822] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.217s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 926.941266] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 926.942713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 801bd73b4d7241af8ce186c2028ad67c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.943664] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.144s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 926.945020] env[62736]: INFO nova.compute.claims [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 926.946726] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 5be949c1e52c438c83baed8d63761523 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 926.978991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 801bd73b4d7241af8ce186c2028ad67c [ 926.986435] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a8d3c11c04d94742b24a7e138779cba8 [ 926.995807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5be949c1e52c438c83baed8d63761523 [ 927.097825] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 927.099691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg af3e0f71694c47f19d9ae3fece016e66 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.134023] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af3e0f71694c47f19d9ae3fece016e66 [ 927.281985] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Releasing lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 927.282590] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 927.282862] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 927.284220] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-45930d09-a653-465c-a71a-625466c2d753 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.287607] env[62736]: DEBUG nova.compute.manager [req-24a08dc5-45d4-46d1-98a8-1558200a700b req-b9c44fe9-47d1-439e-acff-a0cc350e60c0 service nova] [instance: 6add8218-734e-4181-8ada-4657331024e5] Received event network-vif-deleted-c6993fc6-8b41-4675-8af3-4c43f6501e88 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 927.295203] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2785e4e5-d6f3-4afd-a9c6-63805a57a566 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 927.318637] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 6add8218-734e-4181-8ada-4657331024e5 could not be found. [ 927.318834] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 927.319002] env[62736]: INFO nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Took 0.04 seconds to destroy the instance on the hypervisor. [ 927.319224] env[62736]: DEBUG oslo.service.loopingcall [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 927.319520] env[62736]: DEBUG nova.compute.manager [-] [instance: 6add8218-734e-4181-8ada-4657331024e5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 927.319627] env[62736]: DEBUG nova.network.neutron [-] [instance: 6add8218-734e-4181-8ada-4657331024e5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 927.343110] env[62736]: DEBUG nova.network.neutron [-] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 927.343589] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg eddd4701552b48409c1c6a394dd21f93 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.353454] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eddd4701552b48409c1c6a394dd21f93 [ 927.446333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 7c8b44b0b4954437b8c495ced8c3e5f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.449741] env[62736]: DEBUG nova.compute.utils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 927.450326] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 7030188d7a4d44aea2ed5c2badc14c9a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.452283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 765893cd01084c798e6bcaafd16b732c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.453033] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 927.453366] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 927.474093] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 765893cd01084c798e6bcaafd16b732c [ 927.486355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7030188d7a4d44aea2ed5c2badc14c9a [ 927.491672] env[62736]: DEBUG nova.policy [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f9be2719bf36444a9255ca8c15292c05', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e1192ea399b74ef29c41ac66a6e23bca', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 927.495317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c8b44b0b4954437b8c495ced8c3e5f3 [ 927.619805] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 927.773794] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Successfully created port: 0a528b85-3544-4be3-abfa-b5cfea3ebc4f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 927.845845] env[62736]: DEBUG nova.network.neutron [-] [instance: 6add8218-734e-4181-8ada-4657331024e5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 927.846347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 62ddfb2a2cfd42e9a18fbc2d93d6cb41 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.855582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62ddfb2a2cfd42e9a18fbc2d93d6cb41 [ 927.961878] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 927.968445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 898ff57522eb466eb2f7037fcdd27d8a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 927.974638] env[62736]: INFO nova.scheduler.client.report [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance c1001bab-08b9-4bb9-b737-ba7fa5f929f0 [ 927.975660] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 27e2a03015fd4d43978f52f9f3ca37f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 928.004834] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27e2a03015fd4d43978f52f9f3ca37f9 [ 928.008322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 898ff57522eb466eb2f7037fcdd27d8a [ 928.066721] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a0e995d-a37e-451a-a734-7eb7a5469793 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.074838] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b83c42f3-00c7-4116-8a0c-cdb28ca33dcc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.103942] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9856b71-790c-4a8e-84e9-a86a9874a0a7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.111823] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bff58dfb-d4e6-4427-a47c-c056def284b2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 928.127440] env[62736]: DEBUG nova.compute.provider_tree [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 928.127927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg fde000c8eff1431bacea254c8ec9ee0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 928.135470] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fde000c8eff1431bacea254c8ec9ee0a [ 928.348274] env[62736]: INFO nova.compute.manager [-] [instance: 6add8218-734e-4181-8ada-4657331024e5] Took 1.03 seconds to deallocate network for instance. [ 928.350694] env[62736]: DEBUG nova.compute.claims [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 928.350871] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 928.472906] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 0cfcb95c8b954b7481ecce510551d8d1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 928.478150] env[62736]: DEBUG oslo_concurrency.lockutils [None req-48b8b4a5-2b80-41af-8c09-a99e1346c60b tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "c1001bab-08b9-4bb9-b737-ba7fa5f929f0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 62.818s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 928.501765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cfcb95c8b954b7481ecce510551d8d1 [ 928.631544] env[62736]: DEBUG nova.scheduler.client.report [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 928.634130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 6ab3f01db018494381057bc16e917607 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 928.648240] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ab3f01db018494381057bc16e917607 [ 928.692700] env[62736]: ERROR nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 928.692700] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 928.692700] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 928.692700] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 928.692700] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 928.692700] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 928.692700] env[62736]: ERROR nova.compute.manager raise self.value [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 928.692700] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 928.692700] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 928.692700] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 928.693251] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 928.693251] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 928.693251] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 928.693251] env[62736]: ERROR nova.compute.manager [ 928.693251] env[62736]: Traceback (most recent call last): [ 928.693251] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 928.693251] env[62736]: listener.cb(fileno) [ 928.693251] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 928.693251] env[62736]: result = function(*args, **kwargs) [ 928.693251] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 928.693251] env[62736]: return func(*args, **kwargs) [ 928.693251] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 928.693251] env[62736]: raise e [ 928.693251] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 928.693251] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 928.693251] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 928.693251] env[62736]: created_port_ids = self._update_ports_for_instance( [ 928.693251] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 928.693251] env[62736]: with excutils.save_and_reraise_exception(): [ 928.693251] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 928.693251] env[62736]: self.force_reraise() [ 928.693251] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 928.693251] env[62736]: raise self.value [ 928.693251] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 928.693251] env[62736]: updated_port = self._update_port( [ 928.693251] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 928.693251] env[62736]: _ensure_no_port_binding_failure(port) [ 928.693251] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 928.693251] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 928.694147] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 928.694147] env[62736]: Removing descriptor: 17 [ 928.975091] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 928.998893] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 928.999155] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 928.999453] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 928.999658] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 928.999834] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 929.000032] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 929.000306] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 929.000486] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 929.001003] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 929.001003] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 929.001132] env[62736]: DEBUG nova.virt.hardware [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 929.001999] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f30242bf-56b0-4454-92a4-f6057328c3fd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.010743] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bc86935-520d-41fb-83a6-986d19d43411 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 929.024459] env[62736]: ERROR nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Traceback (most recent call last): [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] yield resources [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.driver.spawn(context, instance, image_meta, [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] vm_ref = self.build_virtual_machine(instance, [ 929.024459] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] vif_infos = vmwarevif.get_vif_info(self._session, [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] for vif in network_info: [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return self._sync_wrapper(fn, *args, **kwargs) [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.wait() [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self[:] = self._gt.wait() [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return self._exit_event.wait() [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 929.024836] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] current.throw(*self._exc) [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] result = function(*args, **kwargs) [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return func(*args, **kwargs) [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise e [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] nwinfo = self.network_api.allocate_for_instance( [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] created_port_ids = self._update_ports_for_instance( [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] with excutils.save_and_reraise_exception(): [ 929.025287] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.force_reraise() [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise self.value [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] updated_port = self._update_port( [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] _ensure_no_port_binding_failure(port) [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise exception.PortBindingFailed(port_id=port['id']) [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 929.025688] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] [ 929.025688] env[62736]: INFO nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Terminating instance [ 929.026697] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquiring lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 929.026855] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquired lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 929.027017] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 929.027463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg dcf3f3df472f4292ab92342dbc3181ff in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.033955] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dcf3f3df472f4292ab92342dbc3181ff [ 929.137048] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.193s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.137660] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 929.139539] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg fccc0c092d9c444095419341105481c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.141014] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.631s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.142992] env[62736]: INFO nova.compute.claims [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 929.144854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 83c78a7ce1654105a5ad9b447408a9e9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.175763] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fccc0c092d9c444095419341105481c1 [ 929.181518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83c78a7ce1654105a5ad9b447408a9e9 [ 929.311120] env[62736]: DEBUG nova.compute.manager [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Received event network-changed-0a528b85-3544-4be3-abfa-b5cfea3ebc4f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 929.311320] env[62736]: DEBUG nova.compute.manager [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Refreshing instance network info cache due to event network-changed-0a528b85-3544-4be3-abfa-b5cfea3ebc4f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 929.311518] env[62736]: DEBUG oslo_concurrency.lockutils [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] Acquiring lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 929.543123] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 929.620372] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 929.620948] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg b0cd13f7582f41e894d62d7e93d17e0e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.629290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b0cd13f7582f41e894d62d7e93d17e0e [ 929.646846] env[62736]: DEBUG nova.compute.utils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 929.647559] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 328e24d5d2f9436cb63fd923857c1f8f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.649815] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg cb15bf0c8dc6457fa9cb7491de3cb204 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.650735] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 929.650988] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 929.656929] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cb15bf0c8dc6457fa9cb7491de3cb204 [ 929.658738] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 328e24d5d2f9436cb63fd923857c1f8f [ 929.687998] env[62736]: DEBUG nova.policy [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'cd65b3ab163145e1bf370aa0dc63f65c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '8003d16b1cc048bc94407d028838a4f6', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 929.902186] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "65fc24cb-71af-4ca3-8e46-b010a79a65be" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.902507] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "65fc24cb-71af-4ca3-8e46-b010a79a65be" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.903033] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg c2a2dc0df5924a2899a4b4b97383dd9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 929.911845] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2a2dc0df5924a2899a4b4b97383dd9f [ 930.007648] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Successfully created port: ad725b37-2130-4767-be14-161ff994714f {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 930.123437] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Releasing lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 930.123878] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 930.124085] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 930.124401] env[62736]: DEBUG oslo_concurrency.lockutils [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] Acquired lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 930.124571] env[62736]: DEBUG nova.network.neutron [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Refreshing network info cache for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 930.125014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] Expecting reply to msg b3e397c405a24f4a9bbf264200d2890f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.125808] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e795709a-5028-4512-9c63-5e699e252bc4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.132938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3e397c405a24f4a9bbf264200d2890f [ 930.137225] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-950925dc-5e99-47cb-9c03-6e6a02f7f8fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.151640] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 930.153544] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 9959644c87f34ebe8744a103c095217e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.164891] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 41e8e7c0-a518-43bc-82fa-5f77704f4fc1 could not be found. [ 930.165023] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 930.165198] env[62736]: INFO nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 930.165447] env[62736]: DEBUG oslo.service.loopingcall [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 930.165665] env[62736]: DEBUG nova.compute.manager [-] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 930.165757] env[62736]: DEBUG nova.network.neutron [-] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 930.190267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9959644c87f34ebe8744a103c095217e [ 930.197782] env[62736]: DEBUG nova.network.neutron [-] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 930.198275] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 22cfedf1673c46f497a5b335a9ffa572 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.205306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 22cfedf1673c46f497a5b335a9ffa572 [ 930.268407] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba523f8e-e1eb-4190-93f6-17edf32b716f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.276420] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2619bc5-dd20-475e-84ad-2044305e1405 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.305597] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b96a41e4-c267-41b1-82ee-b7f78f773def {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.312829] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-756782b0-b4b3-4ffb-92a3-627ae999f7dd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 930.326869] env[62736]: DEBUG nova.compute.provider_tree [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 930.327335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg a6f0c3a67780439987f35c8c42d04fb8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.334849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6f0c3a67780439987f35c8c42d04fb8 [ 930.405742] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 930.407997] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 3bc5b201d6cb489c9f5461a51cee329b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.440521] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3bc5b201d6cb489c9f5461a51cee329b [ 930.654758] env[62736]: DEBUG nova.network.neutron [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 930.663973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 7df29bafd98f4eeca93fc53569534293 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.699617] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7df29bafd98f4eeca93fc53569534293 [ 930.702701] env[62736]: DEBUG nova.network.neutron [-] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 930.703102] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 760db453ac104c989b979e1b183b57dc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.712277] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 760db453ac104c989b979e1b183b57dc [ 930.807736] env[62736]: DEBUG nova.network.neutron [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 930.808290] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] Expecting reply to msg d0bac4ea64e04c5aadc48e99bef73333 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.822949] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0bac4ea64e04c5aadc48e99bef73333 [ 930.830173] env[62736]: DEBUG nova.scheduler.client.report [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 930.832683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 450732c20efd4f67a13cae6d21dd7d21 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 930.845276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 450732c20efd4f67a13cae6d21dd7d21 [ 930.935806] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 931.032432] env[62736]: ERROR nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 931.032432] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.032432] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.032432] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.032432] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.032432] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.032432] env[62736]: ERROR nova.compute.manager raise self.value [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.032432] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 931.032432] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.032432] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 931.032911] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.032911] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 931.032911] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 931.032911] env[62736]: ERROR nova.compute.manager [ 931.032911] env[62736]: Traceback (most recent call last): [ 931.032911] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 931.032911] env[62736]: listener.cb(fileno) [ 931.032911] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 931.032911] env[62736]: result = function(*args, **kwargs) [ 931.032911] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 931.032911] env[62736]: return func(*args, **kwargs) [ 931.032911] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 931.032911] env[62736]: raise e [ 931.032911] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.032911] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 931.032911] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.032911] env[62736]: created_port_ids = self._update_ports_for_instance( [ 931.032911] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.032911] env[62736]: with excutils.save_and_reraise_exception(): [ 931.032911] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.032911] env[62736]: self.force_reraise() [ 931.032911] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.032911] env[62736]: raise self.value [ 931.032911] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.032911] env[62736]: updated_port = self._update_port( [ 931.032911] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.032911] env[62736]: _ensure_no_port_binding_failure(port) [ 931.032911] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.032911] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 931.033673] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 931.033673] env[62736]: Removing descriptor: 17 [ 931.166808] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 931.194094] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 931.194347] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 931.194500] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 931.194676] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 931.194814] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 931.194952] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 931.195147] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 931.195299] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 931.195500] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 931.195664] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 931.195827] env[62736]: DEBUG nova.virt.hardware [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 931.196835] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ef0ac8a0-15bb-4863-bba8-964604b89937 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.205370] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e10367cb-468d-4f3b-a353-ca0b14efec83 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.218399] env[62736]: INFO nova.compute.manager [-] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Took 1.05 seconds to deallocate network for instance. [ 931.220706] env[62736]: DEBUG nova.compute.claims [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 931.220887] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 931.231043] env[62736]: ERROR nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Traceback (most recent call last): [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] yield resources [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.driver.spawn(context, instance, image_meta, [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self._vmops.spawn(context, instance, image_meta, injected_files, [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] vm_ref = self.build_virtual_machine(instance, [ 931.231043] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] vif_infos = vmwarevif.get_vif_info(self._session, [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] for vif in network_info: [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return self._sync_wrapper(fn, *args, **kwargs) [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.wait() [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self[:] = self._gt.wait() [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return self._exit_event.wait() [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 931.231671] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] current.throw(*self._exc) [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] result = function(*args, **kwargs) [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return func(*args, **kwargs) [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise e [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] nwinfo = self.network_api.allocate_for_instance( [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] created_port_ids = self._update_ports_for_instance( [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] with excutils.save_and_reraise_exception(): [ 931.232310] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.force_reraise() [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise self.value [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] updated_port = self._update_port( [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] _ensure_no_port_binding_failure(port) [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise exception.PortBindingFailed(port_id=port['id']) [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 931.232892] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] [ 931.232892] env[62736]: INFO nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Terminating instance [ 931.233469] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 931.233469] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 931.233625] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 931.233926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg e34732ab14444bd88d5d07bf76641dd6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 931.240471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e34732ab14444bd88d5d07bf76641dd6 [ 931.310508] env[62736]: DEBUG oslo_concurrency.lockutils [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] Releasing lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 931.310944] env[62736]: DEBUG nova.compute.manager [req-f8c06647-1aa8-472e-a9a1-8b80e86f2f15 req-214c1a36-5257-4a22-a184-1d1da2a0df22 service nova] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Received event network-vif-deleted-0a528b85-3544-4be3-abfa-b5cfea3ebc4f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 931.335702] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.195s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 931.336711] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 931.338671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 04ce0ef6ce354254a84ead71e894919f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 931.340541] env[62736]: DEBUG nova.compute.manager [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Received event network-changed-ad725b37-2130-4767-be14-161ff994714f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 931.340851] env[62736]: DEBUG nova.compute.manager [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Refreshing instance network info cache due to event network-changed-ad725b37-2130-4767-be14-161ff994714f. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 931.341188] env[62736]: DEBUG oslo_concurrency.lockutils [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] Acquiring lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 931.341691] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.226s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 931.343557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a669761eee7d4748b75800b6c218d9a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 931.371608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04ce0ef6ce354254a84ead71e894919f [ 931.373425] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a669761eee7d4748b75800b6c218d9a7 [ 931.752119] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 931.843147] env[62736]: DEBUG nova.compute.utils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 931.843912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 7f8103fb03754e77b8294aa1521e39b2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 931.845555] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 931.845831] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 931.856471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7f8103fb03754e77b8294aa1521e39b2 [ 931.876596] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 931.877094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 3dbde6557587403a89975bba5f874238 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 931.887869] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3dbde6557587403a89975bba5f874238 [ 931.924397] env[62736]: DEBUG nova.policy [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '246365bf1682441ab86b7b37c2f97971', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '96e09f148ccf440e856165d451068702', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 931.988575] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de697c19-482a-4e3b-87d1-a89f3b5acde8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 931.997119] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-648dad98-1148-4183-920d-6825953fe406 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.033986] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8aa434df-b331-434f-adf6-99388b659e36 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.042394] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-757d84bf-cab1-440f-831e-5ac9df87e007 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.055941] env[62736]: DEBUG nova.compute.provider_tree [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 932.056523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 89d2cb1c2e014e909b50d198b6dc6179 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.063790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89d2cb1c2e014e909b50d198b6dc6179 [ 932.346422] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 932.348733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 2fa5f4b109294db2889784d283d23a5e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.388932] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 932.388932] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 932.389069] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 932.390466] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2fa5f4b109294db2889784d283d23a5e [ 932.391744] env[62736]: DEBUG oslo_concurrency.lockutils [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] Acquired lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 932.391744] env[62736]: DEBUG nova.network.neutron [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Refreshing network info cache for port ad725b37-2130-4767-be14-161ff994714f {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 932.391744] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] Expecting reply to msg 4d4cdc7d843942a8a05a010416d673e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.392220] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7a74ef36-3661-44be-916c-add3e74b32d8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.398174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d4cdc7d843942a8a05a010416d673e6 [ 932.403535] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e00665f-52b3-493e-9879-0e60439f4bca {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.432277] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d02ada4d-6801-4353-a664-779c3cd5f163 could not be found. [ 932.436364] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 932.436696] env[62736]: INFO nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Took 0.05 seconds to destroy the instance on the hypervisor. [ 932.437095] env[62736]: DEBUG oslo.service.loopingcall [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 932.437990] env[62736]: DEBUG nova.compute.manager [-] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 932.437990] env[62736]: DEBUG nova.network.neutron [-] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 932.468661] env[62736]: DEBUG nova.network.neutron [-] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 932.469186] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 778dabeeb8354927a752b267c8317019 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.472608] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Successfully created port: 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 932.475990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 778dabeeb8354927a752b267c8317019 [ 932.559010] env[62736]: DEBUG nova.scheduler.client.report [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 932.561806] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 96077b75a04d496b9247eb85139665fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.573403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96077b75a04d496b9247eb85139665fa [ 932.854667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 7035a694aabf41dbb1924f5d7f2e841e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.900305] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7035a694aabf41dbb1924f5d7f2e841e [ 932.915761] env[62736]: DEBUG nova.network.neutron [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 932.972272] env[62736]: DEBUG nova.network.neutron [-] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 932.972708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 1f66406a79ae496ca5e3dc79a2be6874 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 932.983334] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f66406a79ae496ca5e3dc79a2be6874 [ 933.041033] env[62736]: DEBUG nova.network.neutron [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.041580] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] Expecting reply to msg 9af4e34e1f4a4c82a82518eb27bfe0be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.049856] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9af4e34e1f4a4c82a82518eb27bfe0be [ 933.065081] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.723s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 933.065831] env[62736]: ERROR nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Traceback (most recent call last): [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.driver.spawn(context, instance, image_meta, [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] vm_ref = self.build_virtual_machine(instance, [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] vif_infos = vmwarevif.get_vif_info(self._session, [ 933.065831] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] for vif in network_info: [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return self._sync_wrapper(fn, *args, **kwargs) [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.wait() [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self[:] = self._gt.wait() [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return self._exit_event.wait() [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] current.throw(*self._exc) [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 933.066229] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] result = function(*args, **kwargs) [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] return func(*args, **kwargs) [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise e [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] nwinfo = self.network_api.allocate_for_instance( [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] created_port_ids = self._update_ports_for_instance( [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] with excutils.save_and_reraise_exception(): [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] self.force_reraise() [ 933.066729] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise self.value [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] updated_port = self._update_port( [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] _ensure_no_port_binding_failure(port) [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] raise exception.PortBindingFailed(port_id=port['id']) [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] nova.exception.PortBindingFailed: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. [ 933.067555] env[62736]: ERROR nova.compute.manager [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] [ 933.067555] env[62736]: DEBUG nova.compute.utils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 933.068046] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 8.052s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 933.068260] env[62736]: DEBUG nova.objects.instance [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lazy-loading 'resources' on Instance uuid e7d784c2-53f8-4da7-8c25-4bdf5362e94d {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 933.068984] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 6ee98b59324f4a9b87cc2dd06a2cd30d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.070478] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Build of instance c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5 was re-scheduled: Binding failed for port 3a7327d4-0399-4b0e-ba0d-a4cab0550f4f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 933.070985] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 933.071226] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 933.071400] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 933.071590] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 933.071994] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg df53b63571664d2caf9cb3d483ddabf1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.076467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ee98b59324f4a9b87cc2dd06a2cd30d [ 933.079332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg df53b63571664d2caf9cb3d483ddabf1 [ 933.172239] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquiring lock "75d0ffcb-0478-441d-a2dd-0429d5d03055" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 933.172522] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "75d0ffcb-0478-441d-a2dd-0429d5d03055" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 933.172909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg dcd5ac64164f4bf581a4bcdca6cd5886 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.181979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dcd5ac64164f4bf581a4bcdca6cd5886 [ 933.358234] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 933.368817] env[62736]: DEBUG nova.compute.manager [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Received event network-changed-4a9dd839-ec2d-49fa-a097-d4407ef9bc7e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 933.368907] env[62736]: DEBUG nova.compute.manager [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Refreshing instance network info cache due to event network-changed-4a9dd839-ec2d-49fa-a097-d4407ef9bc7e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 933.369096] env[62736]: DEBUG oslo_concurrency.lockutils [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] Acquiring lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 933.369239] env[62736]: DEBUG oslo_concurrency.lockutils [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] Acquired lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 933.369394] env[62736]: DEBUG nova.network.neutron [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Refreshing network info cache for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 933.369841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] Expecting reply to msg 83860b5011674b86b9048cb5b6f0c3aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.382701] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 933.382949] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 933.383102] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 933.383279] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 933.383420] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 933.383563] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 933.383771] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 933.383923] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 933.384103] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 933.384269] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 933.384811] env[62736]: DEBUG nova.virt.hardware [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 933.385643] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cde1472-9a4b-4ce9-87cb-495b8e8aea41 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.388670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83860b5011674b86b9048cb5b6f0c3aa [ 933.395590] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38d49e09-8dbd-4f15-8b4f-fd3d62d2ffe3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.476386] env[62736]: INFO nova.compute.manager [-] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Took 1.04 seconds to deallocate network for instance. [ 933.482590] env[62736]: DEBUG nova.compute.claims [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 933.482801] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 933.543740] env[62736]: DEBUG oslo_concurrency.lockutils [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] Releasing lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 933.544036] env[62736]: DEBUG nova.compute.manager [req-bf459f3c-dff7-4be0-8b7f-4a267dafb91d req-45ac4d29-3afb-48b6-9a19-2a336348658e service nova] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Received event network-vif-deleted-ad725b37-2130-4767-be14-161ff994714f {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 933.676032] env[62736]: ERROR nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 933.676032] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 933.676032] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 933.676032] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 933.676032] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 933.676032] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 933.676032] env[62736]: ERROR nova.compute.manager raise self.value [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 933.676032] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 933.676032] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 933.676032] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 933.676597] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 933.676597] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 933.676597] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 933.676597] env[62736]: ERROR nova.compute.manager [ 933.676597] env[62736]: Traceback (most recent call last): [ 933.676597] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 933.676597] env[62736]: listener.cb(fileno) [ 933.676597] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 933.676597] env[62736]: result = function(*args, **kwargs) [ 933.676597] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 933.676597] env[62736]: return func(*args, **kwargs) [ 933.676597] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 933.676597] env[62736]: raise e [ 933.676597] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 933.676597] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 933.676597] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 933.676597] env[62736]: created_port_ids = self._update_ports_for_instance( [ 933.676597] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 933.676597] env[62736]: with excutils.save_and_reraise_exception(): [ 933.676597] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 933.676597] env[62736]: self.force_reraise() [ 933.676597] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 933.676597] env[62736]: raise self.value [ 933.676597] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 933.676597] env[62736]: updated_port = self._update_port( [ 933.676597] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 933.676597] env[62736]: _ensure_no_port_binding_failure(port) [ 933.676597] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 933.676597] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 933.677664] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 933.677664] env[62736]: Removing descriptor: 19 [ 933.677664] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 933.678874] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 933.680706] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 1aa84d1ba1464947bceec9ff53cb1c51 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.682492] env[62736]: ERROR nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Traceback (most recent call last): [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] yield resources [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.driver.spawn(context, instance, image_meta, [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self._vmops.spawn(context, instance, image_meta, injected_files, [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] vm_ref = self.build_virtual_machine(instance, [ 933.682492] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] vif_infos = vmwarevif.get_vif_info(self._session, [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] for vif in network_info: [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self._sync_wrapper(fn, *args, **kwargs) [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.wait() [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self[:] = self._gt.wait() [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self._exit_event.wait() [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 933.683140] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] result = hub.switch() [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self.greenlet.switch() [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] result = function(*args, **kwargs) [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return func(*args, **kwargs) [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise e [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] nwinfo = self.network_api.allocate_for_instance( [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] created_port_ids = self._update_ports_for_instance( [ 933.683565] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] with excutils.save_and_reraise_exception(): [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.force_reraise() [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise self.value [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] updated_port = self._update_port( [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] _ensure_no_port_binding_failure(port) [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise exception.PortBindingFailed(port_id=port['id']) [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 933.683975] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] [ 933.684412] env[62736]: INFO nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Terminating instance [ 933.686496] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquiring lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 933.706465] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86eb827f-7c99-4c7b-820e-f6e8aced4902 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.714998] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac1770d1-e0b9-4cec-a63d-5b3cefdb3835 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.719070] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1aa84d1ba1464947bceec9ff53cb1c51 [ 933.750707] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-740e8155-2b4b-4676-b70e-f78b33ae0d67 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.758770] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4d2a2e1-8a44-4ec5-9658-4b42028f6880 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.773413] env[62736]: DEBUG nova.compute.provider_tree [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 933.773917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg c50dfd081d2643b0951cceaa10bd1f6b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.781217] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c50dfd081d2643b0951cceaa10bd1f6b [ 933.786490] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.786998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 660cf63f7fc4463a86fb77be00b023a6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.796062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 660cf63f7fc4463a86fb77be00b023a6 [ 933.890833] env[62736]: DEBUG nova.network.neutron [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 933.978872] env[62736]: DEBUG nova.network.neutron [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 933.979578] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] Expecting reply to msg f510efda3558438fac612508e4508de7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 933.987953] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f510efda3558438fac612508e4508de7 [ 934.202327] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 934.278707] env[62736]: DEBUG nova.scheduler.client.report [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 934.281323] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 1ab2343bd6c54a5899e5cff7e28eaa07 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.289056] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 934.289278] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 934.289512] env[62736]: DEBUG nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 934.289685] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 934.292720] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ab2343bd6c54a5899e5cff7e28eaa07 [ 934.314203] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 934.314771] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 90c35b47db474406ba575bbc636cd00f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.321448] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90c35b47db474406ba575bbc636cd00f [ 934.481666] env[62736]: DEBUG oslo_concurrency.lockutils [req-36e95eba-86fe-4369-9b16-d16c75af6a8e req-1703d421-6b68-4b50-8c4f-8e593e714ab0 service nova] Releasing lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 934.482123] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquired lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 934.482632] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 934.483084] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 3fb18d7dd99747fb8a1e32f6b8ac4f1e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.490124] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fb18d7dd99747fb8a1e32f6b8ac4f1e [ 934.783889] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.716s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.786120] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.166s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 934.787604] env[62736]: INFO nova.compute.claims [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 934.789160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ceae44359ae44d10bb0f96967cd15820 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.800540] env[62736]: INFO nova.scheduler.client.report [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Deleted allocations for instance e7d784c2-53f8-4da7-8c25-4bdf5362e94d [ 934.803051] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 3236cb4a228847acb064f55ae4bf1774 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.816668] env[62736]: DEBUG nova.network.neutron [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.816859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a3471b8fa1f14ec5a653ae967db04a18 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 934.826026] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3471b8fa1f14ec5a653ae967db04a18 [ 934.829765] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ceae44359ae44d10bb0f96967cd15820 [ 934.840325] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3236cb4a228847acb064f55ae4bf1774 [ 934.999495] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 935.063908] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 935.064444] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg eea890fda45a4c4d973c7c88227ba16c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.073148] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eea890fda45a4c4d973c7c88227ba16c [ 935.293028] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 09fe341c76074ee4892e88288048fdb9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.301001] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09fe341c76074ee4892e88288048fdb9 [ 935.307024] env[62736]: DEBUG oslo_concurrency.lockutils [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Lock "e7d784c2-53f8-4da7-8c25-4bdf5362e94d" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 14.024s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 935.307333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-4d3f63d8-2b3b-47aa-af74-37934a1cba9e tempest-ServersAaction247Test-462353810 tempest-ServersAaction247Test-462353810-project-member] Expecting reply to msg 8e199e13c164445fb45d65f1348935d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.318922] env[62736]: INFO nova.compute.manager [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5] Took 1.03 seconds to deallocate network for instance. [ 935.320535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg fef3b10c4d794ec68e38c7dd3d86c2f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.321882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e199e13c164445fb45d65f1348935d5 [ 935.350810] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fef3b10c4d794ec68e38c7dd3d86c2f2 [ 935.396744] env[62736]: DEBUG nova.compute.manager [req-2ed21e24-8b5c-449c-989e-843c5254fbaf req-1923a098-c22a-47fa-a7fe-2df3a6811b3d service nova] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Received event network-vif-deleted-4a9dd839-ec2d-49fa-a097-d4407ef9bc7e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 935.566586] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Releasing lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 935.567180] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 935.567511] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 935.567903] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b6c044e4-53b9-48e7-bb9d-3d4a179bb0d3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.578340] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b55e12bc-448f-476e-9983-b5bce07317cc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.599995] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ca7fc450-351a-407f-96c0-21a39d037e54 could not be found. [ 935.600317] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 935.600728] env[62736]: INFO nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Took 0.03 seconds to destroy the instance on the hypervisor. [ 935.601090] env[62736]: DEBUG oslo.service.loopingcall [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 935.601411] env[62736]: DEBUG nova.compute.manager [-] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 935.601615] env[62736]: DEBUG nova.network.neutron [-] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 935.614233] env[62736]: DEBUG nova.network.neutron [-] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 935.614757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 8657c668e5bb4889b40cc6af89124859 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.621477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8657c668e5bb4889b40cc6af89124859 [ 935.824989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg d7da5dacd2d94d3481ad9ed0b6628c9b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.854190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7da5dacd2d94d3481ad9ed0b6628c9b [ 935.893196] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0981c363-fa39-4ee4-a4fe-745a19665561 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.900830] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa62099c-ee3a-4930-b3ff-989cf889d7a3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.938079] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cd72866-c4f1-49e7-ab55-b9588f1e776b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.945547] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c23bbbd-d410-4277-9435-ccdac60546ed {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.960379] env[62736]: DEBUG nova.compute.provider_tree [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 935.960945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 02e9f12b738b46e6b9f4cd1ba52eadc7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 935.968306] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02e9f12b738b46e6b9f4cd1ba52eadc7 [ 936.116450] env[62736]: DEBUG nova.network.neutron [-] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 936.116953] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 59707f95192e4efda18081a045461902 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 936.125404] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59707f95192e4efda18081a045461902 [ 936.345655] env[62736]: INFO nova.scheduler.client.report [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Deleted allocations for instance c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5 [ 936.351516] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 01c255fde8004e66b92406f8d2e7251a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 936.369741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01c255fde8004e66b92406f8d2e7251a [ 936.464483] env[62736]: DEBUG nova.scheduler.client.report [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 936.466851] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg bf786a9d1b3e45739cf14b7e43e7bb97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 936.478483] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bf786a9d1b3e45739cf14b7e43e7bb97 [ 936.619123] env[62736]: INFO nova.compute.manager [-] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Took 1.02 seconds to deallocate network for instance. [ 936.621517] env[62736]: DEBUG nova.compute.claims [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 936.621697] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 936.860342] env[62736]: DEBUG oslo_concurrency.lockutils [None req-9457cabf-f3b3-4003-aea5-6e5df03f1c19 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "c42bf89c-91eb-4ed6-90b3-dbdc7b92b7a5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 68.011s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 936.969620] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.183s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 936.970160] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 936.971819] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 956ae6e329a64b6da2ccb35a5501676d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 936.972839] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.622s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 936.974600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 1f0ee344269d4055b880637b23c8f6e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 937.001990] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 956ae6e329a64b6da2ccb35a5501676d [ 937.004283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f0ee344269d4055b880637b23c8f6e6 [ 937.477863] env[62736]: DEBUG nova.compute.utils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 937.478534] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 9ff5d18c596643518d5c78595da0d021 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 937.482533] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 937.482694] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 937.493862] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ff5d18c596643518d5c78595da0d021 [ 937.535948] env[62736]: DEBUG nova.policy [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 937.581050] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eccc3b77-e98b-4f42-9b7b-87cbe220d567 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.589577] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f298c2ea-9d0a-40ff-b697-abc54d8edb6e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.627172] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-736ff10c-ea26-41be-b13d-e5c4dd053a86 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.635345] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9564159e-2dec-43f8-b917-70da3f77defd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.650661] env[62736]: DEBUG nova.compute.provider_tree [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 937.651388] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 4d0d9e778ce143f4b4f1711e39978b68 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 937.660618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d0d9e778ce143f4b4f1711e39978b68 [ 937.982928] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 937.984734] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 35381f203cde4b809e90d9b181babeec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 937.989636] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Successfully created port: 0534a202-0f8f-45f6-b574-5d9aeeed9133 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 938.019035] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35381f203cde4b809e90d9b181babeec [ 938.153851] env[62736]: DEBUG nova.scheduler.client.report [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 938.156209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg f3c67b3702154c5a9d4fde41b063ffc0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.169397] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3c67b3702154c5a9d4fde41b063ffc0 [ 938.489121] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d34334bbdfbf4333801696d858334137 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.544736] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d34334bbdfbf4333801696d858334137 [ 938.577752] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "d929eb07-7c21-40fe-9de5-da578e068a13" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 938.577980] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "d929eb07-7c21-40fe-9de5-da578e068a13" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 938.578422] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg fc373abd6eb5487eb5d12e9cf976e218 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.586822] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc373abd6eb5487eb5d12e9cf976e218 [ 938.617409] env[62736]: DEBUG nova.compute.manager [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Received event network-changed-0534a202-0f8f-45f6-b574-5d9aeeed9133 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 938.617605] env[62736]: DEBUG nova.compute.manager [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Refreshing instance network info cache due to event network-changed-0534a202-0f8f-45f6-b574-5d9aeeed9133. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 938.617813] env[62736]: DEBUG oslo_concurrency.lockutils [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] Acquiring lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 938.617950] env[62736]: DEBUG oslo_concurrency.lockutils [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] Acquired lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 938.618125] env[62736]: DEBUG nova.network.neutron [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Refreshing network info cache for port 0534a202-0f8f-45f6-b574-5d9aeeed9133 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 938.618594] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] Expecting reply to msg e08763eb5b89439897303773983bf9e1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.624920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e08763eb5b89439897303773983bf9e1 [ 938.658821] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.686s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 938.659527] env[62736]: ERROR nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] Traceback (most recent call last): [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.driver.spawn(context, instance, image_meta, [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] vm_ref = self.build_virtual_machine(instance, [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] vif_infos = vmwarevif.get_vif_info(self._session, [ 938.659527] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] for vif in network_info: [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self._sync_wrapper(fn, *args, **kwargs) [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.wait() [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self[:] = self._gt.wait() [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self._exit_event.wait() [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] result = hub.switch() [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 938.659893] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return self.greenlet.switch() [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] result = function(*args, **kwargs) [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] return func(*args, **kwargs) [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise e [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] nwinfo = self.network_api.allocate_for_instance( [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] created_port_ids = self._update_ports_for_instance( [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] with excutils.save_and_reraise_exception(): [ 938.660258] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] self.force_reraise() [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise self.value [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] updated_port = self._update_port( [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] _ensure_no_port_binding_failure(port) [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] raise exception.PortBindingFailed(port_id=port['id']) [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] nova.exception.PortBindingFailed: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. [ 938.660840] env[62736]: ERROR nova.compute.manager [instance: 6add8218-734e-4181-8ada-4657331024e5] [ 938.661144] env[62736]: DEBUG nova.compute.utils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 938.662103] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 7.726s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 938.663732] env[62736]: INFO nova.compute.claims [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 938.665210] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1ded9652317c4ffd8b1c5f6f77d627ca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.666322] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Build of instance 6add8218-734e-4181-8ada-4657331024e5 was re-scheduled: Binding failed for port c6993fc6-8b41-4675-8af3-4c43f6501e88, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 938.666757] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 938.666983] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquiring lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 938.667124] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Acquired lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 938.667278] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 938.667793] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 6796080a20a346de879c1fd9dc3e5dbb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 938.673687] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6796080a20a346de879c1fd9dc3e5dbb [ 938.701261] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ded9652317c4ffd8b1c5f6f77d627ca [ 938.859752] env[62736]: ERROR nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 938.859752] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 938.859752] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 938.859752] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 938.859752] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 938.859752] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 938.859752] env[62736]: ERROR nova.compute.manager raise self.value [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 938.859752] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 938.859752] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 938.859752] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 938.860255] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 938.860255] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 938.860255] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 938.860255] env[62736]: ERROR nova.compute.manager [ 938.860255] env[62736]: Traceback (most recent call last): [ 938.860255] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 938.860255] env[62736]: listener.cb(fileno) [ 938.860255] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 938.860255] env[62736]: result = function(*args, **kwargs) [ 938.860255] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 938.860255] env[62736]: return func(*args, **kwargs) [ 938.860255] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 938.860255] env[62736]: raise e [ 938.860255] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 938.860255] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 938.860255] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 938.860255] env[62736]: created_port_ids = self._update_ports_for_instance( [ 938.860255] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 938.860255] env[62736]: with excutils.save_and_reraise_exception(): [ 938.860255] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 938.860255] env[62736]: self.force_reraise() [ 938.860255] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 938.860255] env[62736]: raise self.value [ 938.860255] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 938.860255] env[62736]: updated_port = self._update_port( [ 938.860255] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 938.860255] env[62736]: _ensure_no_port_binding_failure(port) [ 938.860255] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 938.860255] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 938.861009] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 938.861009] env[62736]: Removing descriptor: 17 [ 938.992533] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 939.017358] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 939.017717] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 939.017753] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 939.017912] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 939.018057] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 939.018198] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 939.018388] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 939.018544] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 939.018704] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 939.018857] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 939.019021] env[62736]: DEBUG nova.virt.hardware [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 939.019931] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b77c2a6c-11f2-4f71-ad6e-a4fcfcfe25b7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.028985] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c5ae5092-8782-4026-bb86-4f0fc86a1854 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.043504] env[62736]: ERROR nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Traceback (most recent call last): [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] yield resources [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.driver.spawn(context, instance, image_meta, [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self._vmops.spawn(context, instance, image_meta, injected_files, [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] vm_ref = self.build_virtual_machine(instance, [ 939.043504] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] vif_infos = vmwarevif.get_vif_info(self._session, [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] for vif in network_info: [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return self._sync_wrapper(fn, *args, **kwargs) [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.wait() [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self[:] = self._gt.wait() [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return self._exit_event.wait() [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 939.043903] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] current.throw(*self._exc) [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] result = function(*args, **kwargs) [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return func(*args, **kwargs) [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise e [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] nwinfo = self.network_api.allocate_for_instance( [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] created_port_ids = self._update_ports_for_instance( [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] with excutils.save_and_reraise_exception(): [ 939.044271] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.force_reraise() [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise self.value [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] updated_port = self._update_port( [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] _ensure_no_port_binding_failure(port) [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise exception.PortBindingFailed(port_id=port['id']) [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 939.044629] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] [ 939.044629] env[62736]: INFO nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Terminating instance [ 939.045770] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 939.080333] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 939.082357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 748cd4eb52704ec9abf9d240e24c80ae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.114442] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 748cd4eb52704ec9abf9d240e24c80ae [ 939.135230] env[62736]: DEBUG nova.network.neutron [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 939.170429] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 3e773f59ffd44108b52bd5e49fd01add in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.179130] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e773f59ffd44108b52bd5e49fd01add [ 939.205324] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 939.268674] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 939.269172] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 5c6b242c65514bc5bf63fb31518cfbb7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.277091] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5c6b242c65514bc5bf63fb31518cfbb7 [ 939.291590] env[62736]: DEBUG nova.network.neutron [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 939.292162] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] Expecting reply to msg d159e225f8a04aa4bc9a40595a71522c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.300264] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d159e225f8a04aa4bc9a40595a71522c [ 939.602451] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.772797] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Releasing lock "refresh_cache-6add8218-734e-4181-8ada-4657331024e5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 939.773171] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 939.773461] env[62736]: DEBUG nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 939.773733] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 939.786788] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abd8ef97-192a-4239-99d1-0c4061e7244c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.790126] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 939.790684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg b743e4e35ce840ddb6ead097de03c682 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.796208] env[62736]: DEBUG oslo_concurrency.lockutils [req-19b42a2b-7399-46a6-ad41-78a1962fc8be req-2ead6cea-57ff-4b64-bb20-4bda31e48094 service nova] Releasing lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 939.796842] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 939.797020] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 939.797427] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 3d3fd9b00ef04d46900383d8833db63a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.799153] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0783c87a-166a-44f2-a85a-3235cfc81b96 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.804255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b743e4e35ce840ddb6ead097de03c682 [ 939.804724] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d3fd9b00ef04d46900383d8833db63a [ 939.832851] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-affcbf00-49ed-4857-a1a9-4ce667bdef75 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.841025] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1843d01b-0013-4bca-b13e-3f5394e9d20c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 939.854555] env[62736]: DEBUG nova.compute.provider_tree [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 939.855195] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg bd2b1f9b67d44ff59d0f57b7e7a07344 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 939.861802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd2b1f9b67d44ff59d0f57b7e7a07344 [ 940.297923] env[62736]: DEBUG nova.network.neutron [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 940.298416] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg a4fc54c215d1476ca7c4075c7700cb03 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.307411] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4fc54c215d1476ca7c4075c7700cb03 [ 940.320296] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 940.358164] env[62736]: DEBUG nova.scheduler.client.report [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 940.362052] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 8e268069b9eb4752ad649467d515de67 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.376357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e268069b9eb4752ad649467d515de67 [ 940.400135] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 940.400677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 6dbd7b7e7f7e472980c082c0175322bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.408600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6dbd7b7e7f7e472980c082c0175322bf [ 940.649891] env[62736]: DEBUG nova.compute.manager [req-55b171e9-4537-48e6-b42b-89d2d719bbac req-dd1857be-671d-44f2-8e6a-232260f8d103 service nova] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Received event network-vif-deleted-0534a202-0f8f-45f6-b574-5d9aeeed9133 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 940.801468] env[62736]: INFO nova.compute.manager [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] [instance: 6add8218-734e-4181-8ada-4657331024e5] Took 1.03 seconds to deallocate network for instance. [ 940.803348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg 6c03f944a000428cb9d43b276a7aaccd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.837666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c03f944a000428cb9d43b276a7aaccd [ 940.864790] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.203s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 940.865474] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 940.867111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 74708009170b444a9fecb369ac6360ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.869068] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 9.647s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 940.870282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg ce548db041324218952c887188e1d340 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.901276] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 74708009170b444a9fecb369ac6360ea [ 940.903021] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 940.903414] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 940.903590] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 940.904353] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce548db041324218952c887188e1d340 [ 940.904679] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bbaf0ac1-525f-4085-b72a-7fe1bf9a08c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.915196] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2980280-b9ec-447e-9b12-2d16d72e8ce5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 940.937388] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 98ce815b-3e40-424f-bad6-a1b6a49e0a74 could not be found. [ 940.937633] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 940.937803] env[62736]: INFO nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Took 0.03 seconds to destroy the instance on the hypervisor. [ 940.938044] env[62736]: DEBUG oslo.service.loopingcall [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 940.938260] env[62736]: DEBUG nova.compute.manager [-] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 940.938353] env[62736]: DEBUG nova.network.neutron [-] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 940.954120] env[62736]: DEBUG nova.network.neutron [-] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 940.954657] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 379c3642e96746fca4f2ac61db4074fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 940.962415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 379c3642e96746fca4f2ac61db4074fd [ 941.309983] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg cee904c8b3c54e7ead02b1c340d9f3e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.340854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cee904c8b3c54e7ead02b1c340d9f3e7 [ 941.373076] env[62736]: DEBUG nova.compute.utils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 941.373723] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 50fa1b19cdd24f9bb112ad44dc248f15 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.377483] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 941.377650] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 941.383324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 50fa1b19cdd24f9bb112ad44dc248f15 [ 941.423855] env[62736]: DEBUG nova.policy [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 941.456416] env[62736]: DEBUG nova.network.neutron [-] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 941.456854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 730efdc7f048434d9621e006daee1b2b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.466708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 730efdc7f048434d9621e006daee1b2b [ 941.488728] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dc1c1a4-588d-45ea-9145-cefa1c3a9ba9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.497218] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-efc033c5-9a0f-4544-b00f-3a093119a8b2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.529515] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3a18780-ca19-483b-a069-32ffe8db97dd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.537566] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a8bb2b3-f458-4e8e-bac4-6096b6e9809e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 941.553080] env[62736]: DEBUG nova.compute.provider_tree [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 941.553862] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg b095da41f92b4fc2bed48539829fb681 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.562950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b095da41f92b4fc2bed48539829fb681 [ 941.712237] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Successfully created port: ba08710f-f563-4181-b8d2-4bb9884cb79e {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 941.832983] env[62736]: INFO nova.scheduler.client.report [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Deleted allocations for instance 6add8218-734e-4181-8ada-4657331024e5 [ 941.839436] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Expecting reply to msg daa8dc2eaa0d47db8db7c5712cef7444 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.853879] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg daa8dc2eaa0d47db8db7c5712cef7444 [ 941.881394] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 941.882885] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 2dacd7b62a1a47b79a7f3009570703cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 941.928485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dacd7b62a1a47b79a7f3009570703cf [ 941.959516] env[62736]: INFO nova.compute.manager [-] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Took 1.02 seconds to deallocate network for instance. [ 941.961908] env[62736]: DEBUG nova.compute.claims [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 941.962072] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 942.057027] env[62736]: DEBUG nova.scheduler.client.report [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 942.060202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 3675f196c5f94037999bcbb7ce943c30 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 942.070832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3675f196c5f94037999bcbb7ce943c30 [ 942.348246] env[62736]: DEBUG oslo_concurrency.lockutils [None req-b362cc11-cd22-4da0-9533-8f59011c8987 tempest-AttachVolumeTestJSON-1836922524 tempest-AttachVolumeTestJSON-1836922524-project-member] Lock "6add8218-734e-4181-8ada-4657331024e5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 66.124s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 942.387754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg f1dcb82fcab542908ff54d4bbb8948ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 942.422148] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f1dcb82fcab542908ff54d4bbb8948ef [ 942.563031] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.695s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 942.563728] env[62736]: ERROR nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Traceback (most recent call last): [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.driver.spawn(context, instance, image_meta, [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] vm_ref = self.build_virtual_machine(instance, [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] vif_infos = vmwarevif.get_vif_info(self._session, [ 942.563728] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] for vif in network_info: [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return self._sync_wrapper(fn, *args, **kwargs) [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.wait() [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self[:] = self._gt.wait() [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return self._exit_event.wait() [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] current.throw(*self._exc) [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 942.564092] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] result = function(*args, **kwargs) [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] return func(*args, **kwargs) [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise e [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] nwinfo = self.network_api.allocate_for_instance( [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] created_port_ids = self._update_ports_for_instance( [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] with excutils.save_and_reraise_exception(): [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] self.force_reraise() [ 942.564479] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise self.value [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] updated_port = self._update_port( [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] _ensure_no_port_binding_failure(port) [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] raise exception.PortBindingFailed(port_id=port['id']) [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] nova.exception.PortBindingFailed: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. [ 942.564895] env[62736]: ERROR nova.compute.manager [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] [ 942.564895] env[62736]: DEBUG nova.compute.utils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 942.565596] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 9.083s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 942.568514] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 5641c37b49b54c0d92cc603472328397 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 942.569679] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Build of instance 41e8e7c0-a518-43bc-82fa-5f77704f4fc1 was re-scheduled: Binding failed for port 0a528b85-3544-4be3-abfa-b5cfea3ebc4f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 942.570074] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 942.570291] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquiring lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 942.570506] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Acquired lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 942.570627] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 942.570993] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg c3f711be395c4996a8e0df9dff8c154d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 942.585157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3f711be395c4996a8e0df9dff8c154d [ 942.602267] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5641c37b49b54c0d92cc603472328397 [ 942.676701] env[62736]: ERROR nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 942.676701] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 942.676701] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 942.676701] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 942.676701] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 942.676701] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 942.676701] env[62736]: ERROR nova.compute.manager raise self.value [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 942.676701] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 942.676701] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 942.676701] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 942.677237] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 942.677237] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 942.677237] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 942.677237] env[62736]: ERROR nova.compute.manager [ 942.677237] env[62736]: Traceback (most recent call last): [ 942.677237] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 942.677237] env[62736]: listener.cb(fileno) [ 942.677237] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 942.677237] env[62736]: result = function(*args, **kwargs) [ 942.677237] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 942.677237] env[62736]: return func(*args, **kwargs) [ 942.677237] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 942.677237] env[62736]: raise e [ 942.677237] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 942.677237] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 942.677237] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 942.677237] env[62736]: created_port_ids = self._update_ports_for_instance( [ 942.677237] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 942.677237] env[62736]: with excutils.save_and_reraise_exception(): [ 942.677237] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 942.677237] env[62736]: self.force_reraise() [ 942.677237] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 942.677237] env[62736]: raise self.value [ 942.677237] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 942.677237] env[62736]: updated_port = self._update_port( [ 942.677237] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 942.677237] env[62736]: _ensure_no_port_binding_failure(port) [ 942.677237] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 942.677237] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 942.678399] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 942.678399] env[62736]: Removing descriptor: 17 [ 942.690664] env[62736]: DEBUG nova.compute.manager [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Received event network-changed-ba08710f-f563-4181-b8d2-4bb9884cb79e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 942.690880] env[62736]: DEBUG nova.compute.manager [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Refreshing instance network info cache due to event network-changed-ba08710f-f563-4181-b8d2-4bb9884cb79e. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 942.691007] env[62736]: DEBUG oslo_concurrency.lockutils [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] Acquiring lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 942.691145] env[62736]: DEBUG oslo_concurrency.lockutils [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] Acquired lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 942.691301] env[62736]: DEBUG nova.network.neutron [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Refreshing network info cache for port ba08710f-f563-4181-b8d2-4bb9884cb79e {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 942.691735] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] Expecting reply to msg bd4f03f4297f436caed09e9e50d586a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 942.698623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd4f03f4297f436caed09e9e50d586a3 [ 942.890867] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 942.915050] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 942.915310] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 942.915461] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 942.915638] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 942.915777] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 942.915920] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 942.916150] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 942.916292] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 942.916453] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 942.916605] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 942.916800] env[62736]: DEBUG nova.virt.hardware [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 942.917733] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ded234e-523f-49b3-8b0b-a152ba08bf04 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.925562] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ad7bddc-c596-4ef3-baa3-612a0c202d07 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 942.940143] env[62736]: ERROR nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Traceback (most recent call last): [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] yield resources [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.driver.spawn(context, instance, image_meta, [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self._vmops.spawn(context, instance, image_meta, injected_files, [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] vm_ref = self.build_virtual_machine(instance, [ 942.940143] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] vif_infos = vmwarevif.get_vif_info(self._session, [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] for vif in network_info: [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return self._sync_wrapper(fn, *args, **kwargs) [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.wait() [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self[:] = self._gt.wait() [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return self._exit_event.wait() [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 942.940562] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] current.throw(*self._exc) [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] result = function(*args, **kwargs) [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return func(*args, **kwargs) [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise e [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] nwinfo = self.network_api.allocate_for_instance( [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] created_port_ids = self._update_ports_for_instance( [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] with excutils.save_and_reraise_exception(): [ 942.940937] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.force_reraise() [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise self.value [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] updated_port = self._update_port( [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] _ensure_no_port_binding_failure(port) [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise exception.PortBindingFailed(port_id=port['id']) [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 942.941289] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] [ 942.941289] env[62736]: INFO nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Terminating instance [ 942.942435] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 943.098015] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 943.176192] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86033987-77e5-4f12-bbaa-45fbcd5b5d8d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 943.178626] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 943.178991] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 188a4e6651654a649344bd8efd3d3b93 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.191015] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 188a4e6651654a649344bd8efd3d3b93 [ 943.191015] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15b0f865-6c71-45c7-919f-ebaab2d72eb9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 943.223983] env[62736]: DEBUG nova.network.neutron [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 943.226461] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3cf2a2f-3f48-4384-8f2b-96ce926b69c6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 943.234709] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9051a814-2e09-4454-823f-f57f0d807356 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 943.249012] env[62736]: DEBUG nova.compute.provider_tree [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 943.249595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 36e1dcd836f444b4a7c4ebbe5ce87c01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.256671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36e1dcd836f444b4a7c4ebbe5ce87c01 [ 943.318557] env[62736]: DEBUG nova.network.neutron [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 943.319348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] Expecting reply to msg 5ea27274f6194f9c81711b45090631be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.327870] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ea27274f6194f9c81711b45090631be [ 943.691808] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Releasing lock "refresh_cache-41e8e7c0-a518-43bc-82fa-5f77704f4fc1" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 943.692109] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 943.692304] env[62736]: DEBUG nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 943.692473] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 943.709500] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 943.710016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg 6cb0f33b34c1435293af97489307fa44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.716713] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6cb0f33b34c1435293af97489307fa44 [ 943.752252] env[62736]: DEBUG nova.scheduler.client.report [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 943.755599] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg f4661d49da4f4b438a98231de0693a1d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.766803] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4661d49da4f4b438a98231de0693a1d [ 943.828727] env[62736]: DEBUG oslo_concurrency.lockutils [req-f2559187-601d-42f9-8184-62253d9c1406 req-465abe4e-a4b6-44c1-a9cf-d7d71618baba service nova] Releasing lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 943.828853] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 943.828968] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 943.829378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg f41abbb5fd994d2a9d74b6f2d03cd5de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 943.836409] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f41abbb5fd994d2a9d74b6f2d03cd5de [ 944.212435] env[62736]: DEBUG nova.network.neutron [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 944.212985] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg af98703a3dcd4ce59f24e77ce8166545 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.221794] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af98703a3dcd4ce59f24e77ce8166545 [ 944.258349] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.693s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 944.258996] env[62736]: ERROR nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Traceback (most recent call last): [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.driver.spawn(context, instance, image_meta, [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self._vmops.spawn(context, instance, image_meta, injected_files, [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] vm_ref = self.build_virtual_machine(instance, [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] vif_infos = vmwarevif.get_vif_info(self._session, [ 944.258996] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] for vif in network_info: [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return self._sync_wrapper(fn, *args, **kwargs) [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.wait() [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self[:] = self._gt.wait() [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return self._exit_event.wait() [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] current.throw(*self._exc) [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 944.259339] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] result = function(*args, **kwargs) [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] return func(*args, **kwargs) [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise e [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] nwinfo = self.network_api.allocate_for_instance( [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] created_port_ids = self._update_ports_for_instance( [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] with excutils.save_and_reraise_exception(): [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] self.force_reraise() [ 944.259885] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise self.value [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] updated_port = self._update_port( [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] _ensure_no_port_binding_failure(port) [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] raise exception.PortBindingFailed(port_id=port['id']) [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] nova.exception.PortBindingFailed: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. [ 944.260312] env[62736]: ERROR nova.compute.manager [instance: d02ada4d-6801-4353-a664-779c3cd5f163] [ 944.260312] env[62736]: DEBUG nova.compute.utils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 944.261283] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 10.059s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 944.262572] env[62736]: INFO nova.compute.claims [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 944.264316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 3720ee0fbe18493ab6ef3788773f4ea8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.265433] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Build of instance d02ada4d-6801-4353-a664-779c3cd5f163 was re-scheduled: Binding failed for port ad725b37-2130-4767-be14-161ff994714f, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 944.265890] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 944.266076] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquiring lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 944.266218] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Acquired lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 944.266367] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 944.266718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 33ad2ad6460f4967ada7bd7c69d24d97 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.276727] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33ad2ad6460f4967ada7bd7c69d24d97 [ 944.310976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3720ee0fbe18493ab6ef3788773f4ea8 [ 944.363612] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 944.501666] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 944.502222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0b32eb76f1e949cda385c16e0740d2d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.510597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0b32eb76f1e949cda385c16e0740d2d7 [ 944.715691] env[62736]: DEBUG nova.compute.manager [req-0ccdaf31-650a-4401-94d2-858b55e5c00c req-1cc9853d-fa78-4d20-bef8-ad605a4ba6c6 service nova] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Received event network-vif-deleted-ba08710f-f563-4181-b8d2-4bb9884cb79e {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 944.716400] env[62736]: INFO nova.compute.manager [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] [instance: 41e8e7c0-a518-43bc-82fa-5f77704f4fc1] Took 1.02 seconds to deallocate network for instance. [ 944.718181] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg a4df236c9e3142fdbf61542e769bdebc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.748530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4df236c9e3142fdbf61542e769bdebc [ 944.769642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg d554de30258848e98b7d5e49cd091b6d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.776813] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d554de30258848e98b7d5e49cd091b6d [ 944.785075] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 944.854586] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 944.855115] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg b638d2018b7a47ea837c68d18ea610f2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 944.863508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b638d2018b7a47ea837c68d18ea610f2 [ 945.005057] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 945.005690] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 945.005921] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 945.006215] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-422d5532-e24f-449e-bd39-dcd8b69a636d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.015187] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0320dd28-7de6-4748-ad6d-9f7042cc08d8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.036986] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 65fc24cb-71af-4ca3-8e46-b010a79a65be could not be found. [ 945.037199] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 945.037397] env[62736]: INFO nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Took 0.03 seconds to destroy the instance on the hypervisor. [ 945.037674] env[62736]: DEBUG oslo.service.loopingcall [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 945.037914] env[62736]: DEBUG nova.compute.manager [-] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 945.038006] env[62736]: DEBUG nova.network.neutron [-] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 945.051933] env[62736]: DEBUG nova.network.neutron [-] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 945.052412] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d76bf924e90e419c9ceb58b66fff7c1b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.059162] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d76bf924e90e419c9ceb58b66fff7c1b [ 945.222353] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg dff31b88617741b4a7a4b2b437cd8f2c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.253190] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dff31b88617741b4a7a4b2b437cd8f2c [ 945.359057] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Releasing lock "refresh_cache-d02ada4d-6801-4353-a664-779c3cd5f163" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 945.359294] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 945.359498] env[62736]: DEBUG nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 945.359676] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 945.375029] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 945.375618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg cf994ae6e5e041f38f7b583493fb8b54 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.382316] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf994ae6e5e041f38f7b583493fb8b54 [ 945.384934] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7ce7f09-2341-4602-acf1-539f287abf37 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.393985] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-080f6176-6f34-4696-896a-b70dc07e2d21 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.425249] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d60f15e-381a-4ff0-b737-5fcc8a70678b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.432562] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3a52d8d-30c2-48e3-ae0b-3eb4aa9970fb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 945.446183] env[62736]: DEBUG nova.compute.provider_tree [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 945.446721] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 9805c4ef81a24c6e9a7fc1cf322069fa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.454368] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9805c4ef81a24c6e9a7fc1cf322069fa [ 945.554394] env[62736]: DEBUG nova.network.neutron [-] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 945.555012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c8d57a7f5cdb49b18e12ce929db63416 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.563314] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c8d57a7f5cdb49b18e12ce929db63416 [ 945.755708] env[62736]: INFO nova.scheduler.client.report [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Deleted allocations for instance 41e8e7c0-a518-43bc-82fa-5f77704f4fc1 [ 945.777105] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Expecting reply to msg eb4dc2c43d284b4c8448114e16bcd236 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.791012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb4dc2c43d284b4c8448114e16bcd236 [ 945.878603] env[62736]: DEBUG nova.network.neutron [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 945.879111] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg d7c9188bf8754ee7b23b2389ceb88698 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.886970] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7c9188bf8754ee7b23b2389ceb88698 [ 945.955554] env[62736]: DEBUG nova.scheduler.client.report [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 945.958073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg abe5ec9fa70c43848cc66e0bf7feb7a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 945.971340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg abe5ec9fa70c43848cc66e0bf7feb7a1 [ 946.056849] env[62736]: INFO nova.compute.manager [-] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Took 1.02 seconds to deallocate network for instance. [ 946.059052] env[62736]: DEBUG nova.compute.claims [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 946.059223] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 946.280262] env[62736]: DEBUG oslo_concurrency.lockutils [None req-7c1701df-8a8b-4d7b-b280-f68ff6f52c16 tempest-ServersNegativeTestJSON-1043856013 tempest-ServersNegativeTestJSON-1043856013-project-member] Lock "41e8e7c0-a518-43bc-82fa-5f77704f4fc1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 39.168s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 946.381179] env[62736]: INFO nova.compute.manager [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] [instance: d02ada4d-6801-4353-a664-779c3cd5f163] Took 1.02 seconds to deallocate network for instance. [ 946.383038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 628109a442624b3dafde343160e36a82 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 946.413816] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 628109a442624b3dafde343160e36a82 [ 946.467228] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.206s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 946.467698] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 946.469547] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg c6b419fbdc214ec3958ccb7f2bf9c933 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 946.470667] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 9.849s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 946.472401] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 1347d7ebacdc49efbe9866e93a313585 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 946.499081] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c6b419fbdc214ec3958ccb7f2bf9c933 [ 946.502741] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1347d7ebacdc49efbe9866e93a313585 [ 946.887923] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg 72f0b6bea2a94724a96c80f9d1faf80e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 946.917892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72f0b6bea2a94724a96c80f9d1faf80e [ 946.975245] env[62736]: DEBUG nova.compute.utils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 946.976014] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 6347fdf28501496ab979ee44a3726a24 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 946.981794] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 946.981794] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 946.987348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6347fdf28501496ab979ee44a3726a24 [ 947.029161] env[62736]: DEBUG nova.policy [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '244ec553c8df4c1d9789386e00626390', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '37d11ba4c5e44dc69920c21d01aee53d', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 947.064235] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52c96c09-d844-4044-9b55-de908ef61577 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.073685] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-395ab2a7-7f8d-4338-8c16-d1310abeb5c9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.101951] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbba0df3-43f7-4119-af4a-4c0abc14edf0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.110268] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50809bbc-d1e2-455e-a928-6ef323c45c74 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 947.123892] env[62736]: DEBUG nova.compute.provider_tree [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 947.124619] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 20119dd778c849b0b618df07a5d95ee0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 947.133246] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20119dd778c849b0b618df07a5d95ee0 [ 947.340598] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Successfully created port: 8118c02b-1653-43b8-9584-07011b2a208d {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 947.407569] env[62736]: INFO nova.scheduler.client.report [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Deleted allocations for instance d02ada4d-6801-4353-a664-779c3cd5f163 [ 947.413567] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Expecting reply to msg cf1e4c11377d4ca3ac7fc7a90a42ac0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 947.430612] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cf1e4c11377d4ca3ac7fc7a90a42ac0a [ 947.482227] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 947.484410] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg de13520d337d4248afb67dfce7b868c0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 947.518817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg de13520d337d4248afb67dfce7b868c0 [ 947.627530] env[62736]: DEBUG nova.scheduler.client.report [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 947.629998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg bc655523a8f54b5d8a81c7124243fd21 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 947.644882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc655523a8f54b5d8a81c7124243fd21 [ 947.920368] env[62736]: DEBUG oslo_concurrency.lockutils [None req-5e8d561e-1b49-4914-81c6-5c9f1369c579 tempest-AttachVolumeNegativeTest-1702163546 tempest-AttachVolumeNegativeTest-1702163546-project-member] Lock "d02ada4d-6801-4353-a664-779c3cd5f163" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 32.156s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.988179] env[62736]: INFO nova.virt.block_device [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Booting with volume e058b35d-a55c-4cbe-8ab6-8dd59305b59d at /dev/sda [ 948.027216] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-cbbf81b3-39a0-4f43-b9d7-400e45a987c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.036088] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52446ab8-a78c-408d-95d9-afa5d6a758af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.059802] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6e49a84d-79d1-4e3d-9e23-6d4f99d737a0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.067639] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-743135b8-b2f5-4b70-ab98-a83c3f024b37 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.088246] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ee0bdf7-2c46-4e05-9b55-92f19544e64d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.101767] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92733d5c-f031-40b4-82b9-3ad6fea6671f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 948.123558] env[62736]: DEBUG nova.virt.block_device [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating existing volume attachment record: 05b10067-23e1-4bd5-accf-23207b401b15 {{(pid=62736) _volume_attach /opt/stack/nova/nova/virt/block_device.py:666}} [ 948.132654] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.662s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 948.133371] env[62736]: ERROR nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Traceback (most recent call last): [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.driver.spawn(context, instance, image_meta, [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self._vmops.spawn(context, instance, image_meta, injected_files, [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] vm_ref = self.build_virtual_machine(instance, [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] vif_infos = vmwarevif.get_vif_info(self._session, [ 948.133371] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] for vif in network_info: [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self._sync_wrapper(fn, *args, **kwargs) [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.wait() [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self[:] = self._gt.wait() [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self._exit_event.wait() [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] result = hub.switch() [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 948.133776] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return self.greenlet.switch() [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] result = function(*args, **kwargs) [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] return func(*args, **kwargs) [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise e [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] nwinfo = self.network_api.allocate_for_instance( [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] created_port_ids = self._update_ports_for_instance( [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] with excutils.save_and_reraise_exception(): [ 948.134156] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] self.force_reraise() [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise self.value [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] updated_port = self._update_port( [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] _ensure_no_port_binding_failure(port) [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] raise exception.PortBindingFailed(port_id=port['id']) [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] nova.exception.PortBindingFailed: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. [ 948.134577] env[62736]: ERROR nova.compute.manager [instance: ca7fc450-351a-407f-96c0-21a39d037e54] [ 948.134973] env[62736]: DEBUG nova.compute.utils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 948.136048] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 8.533s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 948.138160] env[62736]: INFO nova.compute.claims [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 948.140090] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 776cb3c79dda473aa57c50704f22ecb6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.141240] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Build of instance ca7fc450-351a-407f-96c0-21a39d037e54 was re-scheduled: Binding failed for port 4a9dd839-ec2d-49fa-a097-d4407ef9bc7e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 948.141903] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 948.142133] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquiring lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 948.142278] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Acquired lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 948.142456] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 948.143078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 6bf6525d9dcf4bfab67beb6180b01003 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.151364] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6bf6525d9dcf4bfab67beb6180b01003 [ 948.189601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 776cb3c79dda473aa57c50704f22ecb6 [ 948.272326] env[62736]: DEBUG nova.compute.manager [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Received event network-changed-8118c02b-1653-43b8-9584-07011b2a208d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 948.272326] env[62736]: DEBUG nova.compute.manager [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Refreshing instance network info cache due to event network-changed-8118c02b-1653-43b8-9584-07011b2a208d. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 948.272476] env[62736]: DEBUG oslo_concurrency.lockutils [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] Acquiring lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 948.272672] env[62736]: DEBUG oslo_concurrency.lockutils [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] Acquired lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 948.273313] env[62736]: DEBUG nova.network.neutron [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Refreshing network info cache for port 8118c02b-1653-43b8-9584-07011b2a208d {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 948.273893] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] Expecting reply to msg 742485d6fc2a4eb6a92750de4cd16350 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.280264] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 742485d6fc2a4eb6a92750de4cd16350 [ 948.430512] env[62736]: ERROR nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 948.430512] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 948.430512] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 948.430512] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 948.430512] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 948.430512] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 948.430512] env[62736]: ERROR nova.compute.manager raise self.value [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 948.430512] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 948.430512] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 948.430512] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 948.431478] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 948.431478] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 948.431478] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 948.431478] env[62736]: ERROR nova.compute.manager [ 948.431478] env[62736]: Traceback (most recent call last): [ 948.431478] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 948.431478] env[62736]: listener.cb(fileno) [ 948.431478] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 948.431478] env[62736]: result = function(*args, **kwargs) [ 948.431478] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 948.431478] env[62736]: return func(*args, **kwargs) [ 948.431478] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 948.431478] env[62736]: raise e [ 948.431478] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 948.431478] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 948.431478] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 948.431478] env[62736]: created_port_ids = self._update_ports_for_instance( [ 948.431478] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 948.431478] env[62736]: with excutils.save_and_reraise_exception(): [ 948.431478] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 948.431478] env[62736]: self.force_reraise() [ 948.431478] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 948.431478] env[62736]: raise self.value [ 948.431478] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 948.431478] env[62736]: updated_port = self._update_port( [ 948.431478] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 948.431478] env[62736]: _ensure_no_port_binding_failure(port) [ 948.431478] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 948.431478] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 948.432833] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 948.432833] env[62736]: Removing descriptor: 17 [ 948.653349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 642d9b3ab437400392b76201b73f287f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.662704] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 642d9b3ab437400392b76201b73f287f [ 948.673115] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 948.682944] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg c22205c5a1184024a2914c5e015b4f23 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.696926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c22205c5a1184024a2914c5e015b4f23 [ 948.961861] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 948.962382] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg c01e5ad466884158a1beb2b46f56fe00 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 948.964640] env[62736]: DEBUG nova.network.neutron [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 948.971035] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c01e5ad466884158a1beb2b46f56fe00 [ 949.040476] env[62736]: DEBUG nova.network.neutron [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 949.041011] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] Expecting reply to msg f6be31bc34e74cdb8084b76022277d41 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.050010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f6be31bc34e74cdb8084b76022277d41 [ 949.227605] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 2b502591c84c42bb81cae56659ff0ba7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.234195] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a1c18a63-2c26-436d-b002-721e0ca192ba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.238476] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b502591c84c42bb81cae56659ff0ba7 [ 949.244908] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73087ae1-bc60-4894-9d91-d8bda9b145ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.279378] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84bbac5d-c952-4cf9-83c8-5b0f980305a7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.286611] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30580afc-6905-42d0-a403-24aaa56da79a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 949.301634] env[62736]: DEBUG nova.compute.provider_tree [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 949.302270] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 64b957fddb2b4fb69e3b4c2f83823fde in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.311232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64b957fddb2b4fb69e3b4c2f83823fde [ 949.467748] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Releasing lock "refresh_cache-ca7fc450-351a-407f-96c0-21a39d037e54" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 949.468106] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 949.469018] env[62736]: DEBUG nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 949.469018] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 949.481932] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 949.482508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 2dbae0a7eb204adfa859abe198e9cb8d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.489197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2dbae0a7eb204adfa859abe198e9cb8d [ 949.543051] env[62736]: DEBUG oslo_concurrency.lockutils [req-6c090084-98c5-492a-b36e-c0ad88264b88 req-df0678ea-876b-45c1-9794-bd2b76336c18 service nova] Releasing lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 949.731710] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 90b00f68ddfd48aa9ade0355e6ef469a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.772145] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90b00f68ddfd48aa9ade0355e6ef469a [ 949.804656] env[62736]: DEBUG nova.scheduler.client.report [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 949.807757] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 44a277efde9546e5a47595730020c361 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.817040] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 949.817267] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 949.817449] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 949.817636] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 949.818203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b80a3fe453ef4352a5ca0bfc290937e4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.821602] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44a277efde9546e5a47595730020c361 [ 949.830486] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b80a3fe453ef4352a5ca0bfc290937e4 [ 949.984615] env[62736]: DEBUG nova.network.neutron [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 949.985202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 2b3e3a9c1bee4ebcb3defb3abca6b32f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 949.993456] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b3e3a9c1bee4ebcb3defb3abca6b32f [ 950.235170] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 950.235731] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 950.235939] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 950.236108] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 950.236305] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 950.236451] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 950.236595] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 950.236916] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 950.237006] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 950.237176] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 950.237338] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 950.237508] env[62736]: DEBUG nova.virt.hardware [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 950.238380] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81642f4f-1f16-4159-bbbb-feaa882396ee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.245921] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20823c4f-65ea-4916-a9bd-dfe02e46ee70 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.260769] env[62736]: ERROR nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Traceback (most recent call last): [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] yield resources [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.driver.spawn(context, instance, image_meta, [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self._vmops.spawn(context, instance, image_meta, injected_files, [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] vm_ref = self.build_virtual_machine(instance, [ 950.260769] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] vif_infos = vmwarevif.get_vif_info(self._session, [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] for vif in network_info: [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return self._sync_wrapper(fn, *args, **kwargs) [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.wait() [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self[:] = self._gt.wait() [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return self._exit_event.wait() [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 950.261144] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] current.throw(*self._exc) [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] result = function(*args, **kwargs) [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return func(*args, **kwargs) [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise e [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] nwinfo = self.network_api.allocate_for_instance( [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] created_port_ids = self._update_ports_for_instance( [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] with excutils.save_and_reraise_exception(): [ 950.261532] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.force_reraise() [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise self.value [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] updated_port = self._update_port( [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] _ensure_no_port_binding_failure(port) [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise exception.PortBindingFailed(port_id=port['id']) [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 950.261916] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] [ 950.261916] env[62736]: INFO nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Terminating instance [ 950.263409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquiring lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 950.263409] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquired lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 950.263409] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 950.263777] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 99f68a6ece2c467db6642804d6026f33 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.270255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99f68a6ece2c467db6642804d6026f33 [ 950.296253] env[62736]: DEBUG nova.compute.manager [req-52002c24-8207-45c4-8ad8-f0240bd5a00a req-dabf5d7b-3dd5-42bb-af54-d370c45f9835 service nova] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Received event network-vif-deleted-8118c02b-1653-43b8-9584-07011b2a208d {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 950.310729] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.175s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 950.311189] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 950.312778] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 09d88bfbfa0e42cb87bd3991c9e641a7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.313634] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 8.352s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 950.315880] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d3032213b49f47e585689717083f0716 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.321898] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 950.322082] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 950.322254] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 950.322420] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 950.322592] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 950.322810] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.323006] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.323217] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.323441] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.323649] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.323855] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.324063] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 950.324310] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 950.324735] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 558b770e2f0747b8937505fc7981b7d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.340323] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 558b770e2f0747b8937505fc7981b7d9 [ 950.350379] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 09d88bfbfa0e42cb87bd3991c9e641a7 [ 950.352692] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d3032213b49f47e585689717083f0716 [ 950.487635] env[62736]: INFO nova.compute.manager [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] [instance: ca7fc450-351a-407f-96c0-21a39d037e54] Took 1.02 seconds to deallocate network for instance. [ 950.489671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 3d5f0828ad21480690297e457be73e62 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.520266] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d5f0828ad21480690297e457be73e62 [ 950.797185] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 950.819376] env[62736]: DEBUG nova.compute.utils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 950.820337] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ae734d324e9f417a82c521d8dab37a3a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.824609] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 950.824864] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 950.828522] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 950.831140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae734d324e9f417a82c521d8dab37a3a [ 950.923217] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4a33058-e736-40ab-ae21-ab0d9b7eb32d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.930903] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfefc5a2-c38b-4a38-9c81-78fd1149f628 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.935205] env[62736]: DEBUG nova.policy [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b785ade05de40c88f4bf3ef92e5a25f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '721d9e362be84a33aa7956177650d445', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 950.962346] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-486f4a9b-3e49-470a-95e2-39af3214693a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.970446] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56f9de76-2b66-4f1c-bbfd-2adc77bdc954 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 950.974775] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 950.975244] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 64376298ba9544b38829f66b70d0af13 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.986342] env[62736]: DEBUG nova.compute.provider_tree [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 950.986808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg bcb0dcb64fbb487592b031df08bd09b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.988329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64376298ba9544b38829f66b70d0af13 [ 950.994642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 35621343908f447391f13e51c9d68439 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 950.996349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bcb0dcb64fbb487592b031df08bd09b9 [ 951.022908] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 35621343908f447391f13e51c9d68439 [ 951.211914] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Successfully created port: 7e97d758-e942-48f8-9b95-ba51fac25f7c {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 951.325280] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 951.326920] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 80f93c4e703645c9be75f96a9222c7a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 951.356340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f93c4e703645c9be75f96a9222c7a0 [ 951.477604] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Releasing lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 951.478217] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 951.478786] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8e267bf7-37b4-4d92-8f03-5126d6da9a17 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.487071] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d568bf6-7242-4b29-aa2a-0689cb2dcefa {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.497922] env[62736]: DEBUG nova.scheduler.client.report [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 951.500182] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 9897a9612ce74bb7810cda66e64e7be1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 951.515709] env[62736]: WARNING nova.virt.vmwareapi.driver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance 75d0ffcb-0478-441d-a2dd-0429d5d03055 could not be found. [ 951.515944] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 951.516677] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9897a9612ce74bb7810cda66e64e7be1 [ 951.517066] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fc17dc86-5b46-4100-bf08-f27646903050 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.521590] env[62736]: INFO nova.scheduler.client.report [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Deleted allocations for instance ca7fc450-351a-407f-96c0-21a39d037e54 [ 951.527577] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Expecting reply to msg 3705fb8a8b5342749019ee98c464f627 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 951.533125] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4451d102-0da8-447b-a6d9-dd86447f5362 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 951.543628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3705fb8a8b5342749019ee98c464f627 [ 951.554892] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 75d0ffcb-0478-441d-a2dd-0429d5d03055 could not be found. [ 951.555100] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 951.555269] env[62736]: INFO nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Took 0.08 seconds to destroy the instance on the hypervisor. [ 951.555498] env[62736]: DEBUG oslo.service.loopingcall [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 951.555707] env[62736]: DEBUG nova.compute.manager [-] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 951.555798] env[62736]: DEBUG nova.network.neutron [-] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 951.574073] env[62736]: DEBUG nova.network.neutron [-] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 951.574564] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3b30dad888b040d9b6904bd052539635 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 951.584942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b30dad888b040d9b6904bd052539635 [ 951.831841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 67b1a52cf1d94cc389e12fe1bfae1457 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 951.869239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 67b1a52cf1d94cc389e12fe1bfae1457 [ 952.005494] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.692s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.006114] env[62736]: ERROR nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Traceback (most recent call last): [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.driver.spawn(context, instance, image_meta, [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self._vmops.spawn(context, instance, image_meta, injected_files, [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] vm_ref = self.build_virtual_machine(instance, [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] vif_infos = vmwarevif.get_vif_info(self._session, [ 952.006114] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] for vif in network_info: [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return self._sync_wrapper(fn, *args, **kwargs) [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.wait() [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self[:] = self._gt.wait() [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return self._exit_event.wait() [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] current.throw(*self._exc) [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 952.006506] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] result = function(*args, **kwargs) [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] return func(*args, **kwargs) [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise e [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] nwinfo = self.network_api.allocate_for_instance( [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] created_port_ids = self._update_ports_for_instance( [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] with excutils.save_and_reraise_exception(): [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] self.force_reraise() [ 952.006874] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise self.value [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] updated_port = self._update_port( [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] _ensure_no_port_binding_failure(port) [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] raise exception.PortBindingFailed(port_id=port['id']) [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] nova.exception.PortBindingFailed: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. [ 952.007227] env[62736]: ERROR nova.compute.manager [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] [ 952.007227] env[62736]: DEBUG nova.compute.utils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 952.008593] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Build of instance 98ce815b-3e40-424f-bad6-a1b6a49e0a74 was re-scheduled: Binding failed for port 0534a202-0f8f-45f6-b574-5d9aeeed9133, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 952.008894] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 952.009126] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.009271] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.009454] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 952.009842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 2ec0a21f6fcd4d32ae001ed55cf4dc30 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.019683] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 5.952s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 952.019683] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 27eee946daa64f0388b319bd4568a2b6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.024790] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2ec0a21f6fcd4d32ae001ed55cf4dc30 [ 952.028768] env[62736]: DEBUG oslo_concurrency.lockutils [None req-1a208bc6-08e3-41ac-a8a8-36b9af58a395 tempest-ServersTestJSON-1369522877 tempest-ServersTestJSON-1369522877-project-member] Lock "ca7fc450-351a-407f-96c0-21a39d037e54" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 31.548s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 952.052216] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27eee946daa64f0388b319bd4568a2b6 [ 952.076338] env[62736]: DEBUG nova.network.neutron [-] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 952.076793] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7a30c485266146e3bbdd73f27faa82d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.090874] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a30c485266146e3bbdd73f27faa82d7 [ 952.101230] env[62736]: ERROR nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 952.101230] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.101230] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.101230] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.101230] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.101230] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.101230] env[62736]: ERROR nova.compute.manager raise self.value [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.101230] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 952.101230] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.101230] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 952.101703] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.101703] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 952.101703] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 952.101703] env[62736]: ERROR nova.compute.manager [ 952.101703] env[62736]: Traceback (most recent call last): [ 952.101703] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 952.101703] env[62736]: listener.cb(fileno) [ 952.101703] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 952.101703] env[62736]: result = function(*args, **kwargs) [ 952.101703] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 952.101703] env[62736]: return func(*args, **kwargs) [ 952.101703] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 952.101703] env[62736]: raise e [ 952.101703] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.101703] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 952.101703] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.101703] env[62736]: created_port_ids = self._update_ports_for_instance( [ 952.101703] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.101703] env[62736]: with excutils.save_and_reraise_exception(): [ 952.101703] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.101703] env[62736]: self.force_reraise() [ 952.101703] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.101703] env[62736]: raise self.value [ 952.101703] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.101703] env[62736]: updated_port = self._update_port( [ 952.101703] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.101703] env[62736]: _ensure_no_port_binding_failure(port) [ 952.101703] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.101703] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 952.102497] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 952.102497] env[62736]: Removing descriptor: 17 [ 952.322447] env[62736]: DEBUG nova.compute.manager [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Received event network-changed-7e97d758-e942-48f8-9b95-ba51fac25f7c {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 952.322654] env[62736]: DEBUG nova.compute.manager [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Refreshing instance network info cache due to event network-changed-7e97d758-e942-48f8-9b95-ba51fac25f7c. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 952.322871] env[62736]: DEBUG oslo_concurrency.lockutils [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] Acquiring lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.323010] env[62736]: DEBUG oslo_concurrency.lockutils [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] Acquired lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 952.323598] env[62736]: DEBUG nova.network.neutron [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Refreshing network info cache for port 7e97d758-e942-48f8-9b95-ba51fac25f7c {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 952.324110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] Expecting reply to msg b5b7b524a8464061a2ec3e1c9aca052d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.333854] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5b7b524a8464061a2ec3e1c9aca052d [ 952.334908] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 952.361100] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 952.361309] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 952.361453] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 952.361632] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 952.361780] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 952.361917] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 952.362117] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 952.362271] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 952.362431] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 952.362589] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 952.362757] env[62736]: DEBUG nova.virt.hardware [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 952.363590] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8db9860b-4c80-4b40-b03a-919ead689a3b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.371350] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b4107e1-f833-4830-9035-52fcd95864e5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.384975] env[62736]: ERROR nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Traceback (most recent call last): [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] yield resources [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.driver.spawn(context, instance, image_meta, [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self._vmops.spawn(context, instance, image_meta, injected_files, [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] vm_ref = self.build_virtual_machine(instance, [ 952.384975] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] vif_infos = vmwarevif.get_vif_info(self._session, [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] for vif in network_info: [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return self._sync_wrapper(fn, *args, **kwargs) [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.wait() [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self[:] = self._gt.wait() [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return self._exit_event.wait() [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 952.385658] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] current.throw(*self._exc) [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] result = function(*args, **kwargs) [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return func(*args, **kwargs) [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise e [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] nwinfo = self.network_api.allocate_for_instance( [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] created_port_ids = self._update_ports_for_instance( [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] with excutils.save_and_reraise_exception(): [ 952.386238] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.force_reraise() [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise self.value [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] updated_port = self._update_port( [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] _ensure_no_port_binding_failure(port) [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise exception.PortBindingFailed(port_id=port['id']) [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 952.386896] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] [ 952.386896] env[62736]: INFO nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Terminating instance [ 952.387498] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 952.530054] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 952.576163] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6441c990-cd39-49f6-bec5-abbdd9ef5b29 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.579340] env[62736]: INFO nova.compute.manager [-] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Took 1.02 seconds to deallocate network for instance. [ 952.586127] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e70227f-9522-4b96-8ed0-40a795bc7178 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.623029] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 952.623857] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d424d0de0b2a48dabfdb8618aa62d159 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.625594] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9534349f-6fd4-4c18-bd89-c3feaf9c31af {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.633455] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d424d0de0b2a48dabfdb8618aa62d159 [ 952.634773] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5e98695-0377-444d-b0be-cb100e49c06c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 952.652367] env[62736]: DEBUG nova.compute.provider_tree [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 952.652965] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 78b022585c2941dba7c307b30e44ebd8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.660197] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78b022585c2941dba7c307b30e44ebd8 [ 952.844346] env[62736]: DEBUG nova.network.neutron [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 952.927857] env[62736]: DEBUG nova.network.neutron [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 952.928399] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] Expecting reply to msg f57d2fc5a5574e3c8dd014a777cfeae7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 952.937974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f57d2fc5a5574e3c8dd014a777cfeae7 [ 953.129656] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-98ce815b-3e40-424f-bad6-a1b6a49e0a74" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.129897] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 953.130075] env[62736]: DEBUG nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 953.130239] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 953.137727] env[62736]: INFO nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Took 0.56 seconds to detach 1 volumes for instance. [ 953.139898] env[62736]: DEBUG nova.compute.claims [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 953.139994] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.143840] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.144353] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 2925812d2226494f8400100eac59b25c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.151177] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2925812d2226494f8400100eac59b25c [ 953.155698] env[62736]: DEBUG nova.scheduler.client.report [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 953.158088] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg b9d03b5833624fbba422ff58753e74e4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.168015] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9d03b5833624fbba422ff58753e74e4 [ 953.430867] env[62736]: DEBUG oslo_concurrency.lockutils [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] Releasing lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 953.431308] env[62736]: DEBUG nova.compute.manager [req-a15a56ef-2164-4cbf-bb28-2349fd505a9f req-cb70489a-7c86-4884-9164-eaddaf1463f7 service nova] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Received event network-vif-deleted-7e97d758-e942-48f8-9b95-ba51fac25f7c {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 953.431426] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.431599] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 953.432042] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg c0239f6cea5245c5b30603120fee680f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.438409] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0239f6cea5245c5b30603120fee680f [ 953.646809] env[62736]: DEBUG nova.network.neutron [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.647363] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 8d9d4d64f13242099676a316e601478b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.655645] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d9d4d64f13242099676a316e601478b [ 953.660342] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.649s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.661019] env[62736]: ERROR nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Traceback (most recent call last): [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.driver.spawn(context, instance, image_meta, [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self._vmops.spawn(context, instance, image_meta, injected_files, [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] vm_ref = self.build_virtual_machine(instance, [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] vif_infos = vmwarevif.get_vif_info(self._session, [ 953.661019] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] for vif in network_info: [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return self._sync_wrapper(fn, *args, **kwargs) [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.wait() [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self[:] = self._gt.wait() [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return self._exit_event.wait() [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] current.throw(*self._exc) [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 953.661370] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] result = function(*args, **kwargs) [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] return func(*args, **kwargs) [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise e [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] nwinfo = self.network_api.allocate_for_instance( [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] created_port_ids = self._update_ports_for_instance( [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] with excutils.save_and_reraise_exception(): [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] self.force_reraise() [ 953.661749] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise self.value [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] updated_port = self._update_port( [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] _ensure_no_port_binding_failure(port) [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] raise exception.PortBindingFailed(port_id=port['id']) [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] nova.exception.PortBindingFailed: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. [ 953.662129] env[62736]: ERROR nova.compute.manager [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] [ 953.662129] env[62736]: DEBUG nova.compute.utils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 953.663224] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Build of instance 65fc24cb-71af-4ca3-8e46-b010a79a65be was re-scheduled: Binding failed for port ba08710f-f563-4181-b8d2-4bb9884cb79e, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 953.663532] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 953.663757] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 953.663898] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 953.664063] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 953.664455] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg b52a82426442435391374090f6d62dbc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.665108] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 2.837s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.665281] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 953.665424] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 953.665839] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.526s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 953.668140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 205cbc9a9555448bb4a194dffb6f237a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 953.669716] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4af86907-dabf-43e1-923b-2d953d2cee0f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.678305] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-89704e1f-7253-4c04-b67e-a41924afb0f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.682971] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b52a82426442435391374090f6d62dbc [ 953.693926] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0310f22d-6c24-4604-8145-817c655cdec9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.702013] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc4a4cb4-6346-4c96-82c0-6e0172805899 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 953.705480] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 205cbc9a9555448bb4a194dffb6f237a [ 953.730942] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181702MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 953.731085] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 953.957466] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.063602] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.064196] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 68e92422b6474547b5c52b25f3719f76 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.073549] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68e92422b6474547b5c52b25f3719f76 [ 954.149536] env[62736]: INFO nova.compute.manager [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 98ce815b-3e40-424f-bad6-a1b6a49e0a74] Took 1.02 seconds to deallocate network for instance. [ 954.150929] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 2d2c64a0ce2e4d12bd62222c352b053c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.187145] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.189350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d2c64a0ce2e4d12bd62222c352b053c [ 954.248664] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-361484df-45e7-4be0-964a-87e2760f7d1c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.256090] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2873e24-6d8b-4caa-960b-1eb4a59f154a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.285736] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 954.286243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 64fa79879cf84f1a8c3b4c1f2ff04f9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.292908] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b223313-4a6c-427a-8827-9c76e00fbae0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.295010] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 64fa79879cf84f1a8c3b4c1f2ff04f9f [ 954.296315] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c76f3522-3af9-48c4-9c4e-86fc150a0bbe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.309703] env[62736]: DEBUG nova.compute.provider_tree [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 954.310470] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg a6a63976d4ef42b0bdc38c609c56ba8e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.317829] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6a63976d4ef42b0bdc38c609c56ba8e [ 954.566577] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.567042] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 954.567230] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 954.567633] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4a94907c-be0b-4f3c-b43b-df7daa42fc63 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.575770] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4a61518-754c-458d-8aeb-f187f924f15e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 954.596088] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d929eb07-7c21-40fe-9de5-da578e068a13 could not be found. [ 954.596285] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 954.596457] env[62736]: INFO nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Took 0.03 seconds to destroy the instance on the hypervisor. [ 954.596697] env[62736]: DEBUG oslo.service.loopingcall [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 954.596899] env[62736]: DEBUG nova.compute.manager [-] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 954.596990] env[62736]: DEBUG nova.network.neutron [-] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 954.611511] env[62736]: DEBUG nova.network.neutron [-] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.612066] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg f4135cfebf5f42d7b10e26cdac800b24 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.618785] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f4135cfebf5f42d7b10e26cdac800b24 [ 954.654634] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 0f4e7e0f40ac4d76b7fc0a988604c0df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.683670] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f4e7e0f40ac4d76b7fc0a988604c0df [ 954.788840] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-65fc24cb-71af-4ca3-8e46-b010a79a65be" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 954.789089] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 954.789274] env[62736]: DEBUG nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 954.789541] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 954.804097] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.804631] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a14670b8be1e46ca8bdd1d05271f349e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.811135] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a14670b8be1e46ca8bdd1d05271f349e [ 954.813128] env[62736]: DEBUG nova.scheduler.client.report [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 954.815340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 624f13ff9dc14f2088ee55558f7bbaca in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 954.825817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 624f13ff9dc14f2088ee55558f7bbaca [ 955.114699] env[62736]: DEBUG nova.network.neutron [-] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.115203] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 7c34010a38124657ae26705387ebe91a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.124381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c34010a38124657ae26705387ebe91a [ 955.175135] env[62736]: INFO nova.scheduler.client.report [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance 98ce815b-3e40-424f-bad6-a1b6a49e0a74 [ 955.182158] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d2463b39f51e4a27ba10988e317bed78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.197581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d2463b39f51e4a27ba10988e317bed78 [ 955.306651] env[62736]: DEBUG nova.network.neutron [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.307176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 5b59aa49bf7d42b39e0359d4a05c9675 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.313943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b59aa49bf7d42b39e0359d4a05c9675 [ 955.317070] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.651s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.317676] env[62736]: ERROR nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Traceback (most recent call last): [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.driver.spawn(context, instance, image_meta, [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self._vmops.spawn(context, instance, image_meta, injected_files, [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] vm_ref = self.build_virtual_machine(instance, [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] vif_infos = vmwarevif.get_vif_info(self._session, [ 955.317676] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] for vif in network_info: [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return self._sync_wrapper(fn, *args, **kwargs) [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.wait() [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self[:] = self._gt.wait() [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return self._exit_event.wait() [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] current.throw(*self._exc) [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 955.318075] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] result = function(*args, **kwargs) [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] return func(*args, **kwargs) [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise e [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] nwinfo = self.network_api.allocate_for_instance( [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] created_port_ids = self._update_ports_for_instance( [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] with excutils.save_and_reraise_exception(): [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] self.force_reraise() [ 955.318474] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise self.value [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] updated_port = self._update_port( [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] _ensure_no_port_binding_failure(port) [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] raise exception.PortBindingFailed(port_id=port['id']) [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] nova.exception.PortBindingFailed: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. [ 955.318884] env[62736]: ERROR nova.compute.manager [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] [ 955.318884] env[62736]: DEBUG nova.compute.utils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 955.319396] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 1.588s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 955.320174] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3dd55edc1d7b4e459b31261e9b19f580 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.321204] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Build of instance 75d0ffcb-0478-441d-a2dd-0429d5d03055 was re-scheduled: Binding failed for port 8118c02b-1653-43b8-9584-07011b2a208d, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 955.321606] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 955.321829] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquiring lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 955.321969] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Acquired lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 955.322116] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 955.322457] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 4d4c2f65290645d49739366cec4b05b3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.327783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d4c2f65290645d49739366cec4b05b3 [ 955.333027] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3dd55edc1d7b4e459b31261e9b19f580 [ 955.617948] env[62736]: INFO nova.compute.manager [-] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Took 1.02 seconds to deallocate network for instance. [ 955.620178] env[62736]: DEBUG nova.compute.claims [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 955.620361] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 955.683330] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a45c6be5-e73b-4367-8afb-678632b2df82 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "98ce815b-3e40-424f-bad6-a1b6a49e0a74" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 29.088s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 955.809335] env[62736]: INFO nova.compute.manager [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: 65fc24cb-71af-4ca3-8e46-b010a79a65be] Took 1.02 seconds to deallocate network for instance. [ 955.811091] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 036b4599d6f04870856e8831222608b0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.824535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg ef06b362e4914a3c8d1ccf64d18fd3db in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.835377] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ef06b362e4914a3c8d1ccf64d18fd3db [ 955.840461] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 955.844279] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 036b4599d6f04870856e8831222608b0 [ 955.923737] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.924274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 46f73dacc13d484898ff6eb01ef0f0ac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 955.932678] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46f73dacc13d484898ff6eb01ef0f0ac [ 956.315519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6ca212b5b80942d0a5fa1c7c443d7f12 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.344396] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg b1e429ac4b244dd08619579b872c8227 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.349112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ca212b5b80942d0a5fa1c7c443d7f12 [ 956.355632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1e429ac4b244dd08619579b872c8227 [ 956.426067] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Releasing lock "refresh_cache-75d0ffcb-0478-441d-a2dd-0429d5d03055" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 956.426302] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 956.426482] env[62736]: DEBUG nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 956.426645] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 956.441273] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 956.441814] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 3b186b0a888a40f4ad4e14d1d89d637b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.448468] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3b186b0a888a40f4ad4e14d1d89d637b [ 956.835894] env[62736]: INFO nova.scheduler.client.report [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance 65fc24cb-71af-4ca3-8e46-b010a79a65be [ 956.842408] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 3133226842c04b349132267cfd418ed5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.847161] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 65fc24cb-71af-4ca3-8e46-b010a79a65be has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 956.847740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 393007d8451c448c8ccb683d0f13ed53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.856515] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3133226842c04b349132267cfd418ed5 [ 956.874669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 393007d8451c448c8ccb683d0f13ed53 [ 956.923861] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "fc58f8bf-849a-452a-a1cb-daec65f662a7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.924137] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "fc58f8bf-849a-452a-a1cb-daec65f662a7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.924597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 6c3d8302b4b44e8694538e9e35ca5709 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.932506] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6c3d8302b4b44e8694538e9e35ca5709 [ 956.944189] env[62736]: DEBUG nova.network.neutron [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 956.944616] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg b94bd3022eaf4fd9a1b89f242c1053e3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 956.950966] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b94bd3022eaf4fd9a1b89f242c1053e3 [ 957.345112] env[62736]: DEBUG oslo_concurrency.lockutils [None req-95c7a0ab-fdab-4784-9d6c-9c8d3cff6e54 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "65fc24cb-71af-4ca3-8e46-b010a79a65be" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 27.442s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 957.351264] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 75d0ffcb-0478-441d-a2dd-0429d5d03055 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 957.351264] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance d929eb07-7c21-40fe-9de5-da578e068a13 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 957.351264] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 1 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 957.351264] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=704MB phys_disk=200GB used_disk=1GB total_vcpus=48 used_vcpus=1 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 957.403015] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea9bff42-2888-430d-be47-f036f3fb27f3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.412440] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70e372d3-4b49-4313-933f-03cd56304f15 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.443693] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 957.444511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ce86080e615d4782babb1a229149b211 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 957.453632] env[62736]: INFO nova.compute.manager [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] [instance: 75d0ffcb-0478-441d-a2dd-0429d5d03055] Took 1.02 seconds to deallocate network for instance. [ 957.453632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 03a9a9050f884b49a3f32d4b865c2701 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 957.453632] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-160cb9dc-5ce6-49b6-a074-2695c800b062 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.458180] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-776ee589-8a09-4a3a-b763-348023910637 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 957.473184] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 957.473935] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 63c978e07a674ccaa25223b90658e03b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 957.488614] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03a9a9050f884b49a3f32d4b865c2701 [ 957.489616] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 63c978e07a674ccaa25223b90658e03b [ 957.491405] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ce86080e615d4782babb1a229149b211 [ 957.954499] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 60de6226db27467091eaa51e44f42eaf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 957.963844] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 957.976848] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 957.979113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 7d7cabb7e05c4739b776c73b1285d7aa in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 957.989597] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d7cabb7e05c4739b776c73b1285d7aa [ 957.992123] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60de6226db27467091eaa51e44f42eaf [ 958.478038] env[62736]: INFO nova.scheduler.client.report [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Deleted allocations for instance 75d0ffcb-0478-441d-a2dd-0429d5d03055 [ 958.483762] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 958.483956] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.165s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.485467] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Expecting reply to msg 4df1215719514c7a95ebd0cf9c32f92d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 958.485851] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 2.865s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.487508] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 27717424c67a45cd86227c678d4a67ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 958.508640] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4df1215719514c7a95ebd0cf9c32f92d [ 958.541894] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27717424c67a45cd86227c678d4a67ec [ 958.574555] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "effae28c-cbe1-4656-9a92-9829d929e3d5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.574900] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "effae28c-cbe1-4656-9a92-9829d929e3d5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.575878] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0254f6bfb17f4f058e4650aedab5cb5f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 958.589205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0254f6bfb17f4f058e4650aedab5cb5f [ 958.990001] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e010e025-8ad5-49a2-be8a-b3592d29f5ca tempest-ServersTestBootFromVolume-1046228888 tempest-ServersTestBootFromVolume-1046228888-project-member] Lock "75d0ffcb-0478-441d-a2dd-0429d5d03055" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.817s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 959.044903] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aee5c6c-becf-48ed-97ae-8f0a2511a5ca {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.054303] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-881ca678-9eba-45d9-96da-c9eb7f207b9f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.083633] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 959.085340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 73560befbf384ea7b925944e675983bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 959.086880] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-344780c8-a077-49c8-bcf5-a4cf772296e3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.094269] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba193784-69e9-451e-bf92-76cd1fc54290 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 959.109994] env[62736]: DEBUG nova.compute.provider_tree [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 959.111037] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 1d6570832ec8435c87091d8313f07849 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 959.116566] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73560befbf384ea7b925944e675983bf [ 959.118413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1d6570832ec8435c87091d8313f07849 [ 959.604899] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 959.614325] env[62736]: DEBUG nova.scheduler.client.report [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 959.616355] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 08be5ec7689140fc9021cf51aba282d8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 959.626962] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 08be5ec7689140fc9021cf51aba282d8 [ 960.119077] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.633s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.119798] env[62736]: ERROR nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Traceback (most recent call last): [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.driver.spawn(context, instance, image_meta, [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self._vmops.spawn(context, instance, image_meta, injected_files, [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] vm_ref = self.build_virtual_machine(instance, [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] vif_infos = vmwarevif.get_vif_info(self._session, [ 960.119798] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] for vif in network_info: [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return self._sync_wrapper(fn, *args, **kwargs) [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.wait() [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self[:] = self._gt.wait() [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return self._exit_event.wait() [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] current.throw(*self._exc) [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 960.120252] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] result = function(*args, **kwargs) [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] return func(*args, **kwargs) [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise e [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] nwinfo = self.network_api.allocate_for_instance( [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] created_port_ids = self._update_ports_for_instance( [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] with excutils.save_and_reraise_exception(): [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] self.force_reraise() [ 960.120829] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise self.value [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] updated_port = self._update_port( [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] _ensure_no_port_binding_failure(port) [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] raise exception.PortBindingFailed(port_id=port['id']) [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] nova.exception.PortBindingFailed: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. [ 960.121381] env[62736]: ERROR nova.compute.manager [instance: d929eb07-7c21-40fe-9de5-da578e068a13] [ 960.121381] env[62736]: DEBUG nova.compute.utils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 960.122144] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.158s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.123529] env[62736]: INFO nova.compute.claims [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 960.125128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 077b398064b44eda8af495bc4cbec6bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 960.126714] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Build of instance d929eb07-7c21-40fe-9de5-da578e068a13 was re-scheduled: Binding failed for port 7e97d758-e942-48f8-9b95-ba51fac25f7c, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 960.127261] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 960.127513] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 960.127668] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 960.127826] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 960.128194] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ed712b74339444c88bc3793dd5d251ad in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 960.134596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed712b74339444c88bc3793dd5d251ad [ 960.160537] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 077b398064b44eda8af495bc4cbec6bb [ 960.632811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg f94a67d587fd478d807ba5686205a457 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 960.641646] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f94a67d587fd478d807ba5686205a457 [ 960.663251] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 960.735906] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 960.736437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 76bb3cddae9b406b9d4e582b14b5645b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 960.744554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76bb3cddae9b406b9d4e582b14b5645b [ 961.192666] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-683423e3-b81b-4a39-9d2d-354212898a5e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.200166] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26d4ddf9-d7a0-4406-8f01-217f823a9d20 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.230551] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd1d1088-ef36-416f-98c7-62e869ed0b1d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.238677] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9db81836-408e-4dcb-ae9b-fb16c2185fd3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.243260] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-d929eb07-7c21-40fe-9de5-da578e068a13" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 961.243486] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 961.243666] env[62736]: DEBUG nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 961.243828] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 961.254372] env[62736]: DEBUG nova.compute.provider_tree [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 961.255322] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d9eb0f59fd0c4189a91fbf5ac8530052 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 961.262140] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 961.262861] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 47ee7744261e4a748f0a6d4a489764cf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 961.264289] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9eb0f59fd0c4189a91fbf5ac8530052 [ 961.269185] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 47ee7744261e4a748f0a6d4a489764cf [ 961.757956] env[62736]: DEBUG nova.scheduler.client.report [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 961.760261] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 8b6d61034b2d404884baea05f0e6a681 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 961.765695] env[62736]: DEBUG nova.network.neutron [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 961.766100] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 710c6f15cb8445b79e14d62e8c1681e6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 961.771129] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8b6d61034b2d404884baea05f0e6a681 [ 961.773754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 710c6f15cb8445b79e14d62e8c1681e6 [ 962.262653] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.140s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 962.263176] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 962.264823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 14045b35edaf4b629df39b61b000624b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.265946] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.661s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 962.267131] env[62736]: INFO nova.compute.claims [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 962.268609] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9c935b62288a41e7a509cdcde673e2f9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.269766] env[62736]: INFO nova.compute.manager [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: d929eb07-7c21-40fe-9de5-da578e068a13] Took 1.03 seconds to deallocate network for instance. [ 962.271184] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 7a0dbd371a5f4e06a0b8f993d8463b18 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.305615] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9c935b62288a41e7a509cdcde673e2f9 [ 962.310583] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7a0dbd371a5f4e06a0b8f993d8463b18 [ 962.313752] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 14045b35edaf4b629df39b61b000624b [ 962.771893] env[62736]: DEBUG nova.compute.utils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 962.772558] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 0c115e0fdc3c4c45ac868b8c24ca4680 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.773468] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 962.773643] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 962.777561] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 1a694da554504083a872a36afb2d455c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.780563] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 1f5338b71dc64630a43687f7f60bf742 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 962.784189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a694da554504083a872a36afb2d455c [ 962.806137] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c115e0fdc3c4c45ac868b8c24ca4680 [ 962.809469] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f5338b71dc64630a43687f7f60bf742 [ 962.820510] env[62736]: DEBUG nova.policy [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 963.125119] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Successfully created port: 246ff4b3-1dd3-440f-aafb-c044dc9cb709 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 963.285935] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 963.287942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e16dd2d6cca344f5b3f013322c3ecd16 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.313588] env[62736]: INFO nova.scheduler.client.report [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Deleted allocations for instance d929eb07-7c21-40fe-9de5-da578e068a13 [ 963.321801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a14c4ddc27ec4c569f88c16645dab0cb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.331369] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e16dd2d6cca344f5b3f013322c3ecd16 [ 963.336842] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a14c4ddc27ec4c569f88c16645dab0cb [ 963.337938] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad8594db-9b94-43d2-94d1-7f76ff88227c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.348203] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff9f9772-790b-4fba-ac5a-d1d43cd7aee1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.386059] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-254ee169-53ec-4b24-a55d-f1aeecaa8e1b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.397305] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4065c837-e5c5-4d74-bf26-c378a760ddeb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 963.412066] env[62736]: DEBUG nova.compute.provider_tree [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 963.412641] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6417baf8c2d04018bc9e7b45b538ee77 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.419785] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6417baf8c2d04018bc9e7b45b538ee77 [ 963.792930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg f3dd8376e0d04efab6ff2d8ee4f7736b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.824471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-e29d3081-b077-4a2a-8665-6ddffc6f1321 tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "d929eb07-7c21-40fe-9de5-da578e068a13" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 25.246s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 963.830402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3dd8376e0d04efab6ff2d8ee4f7736b [ 963.915188] env[62736]: DEBUG nova.scheduler.client.report [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 963.917421] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 9da0132a223b4865bb23c73816a88fbb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.937680] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9da0132a223b4865bb23c73816a88fbb [ 963.990898] env[62736]: DEBUG nova.compute.manager [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Received event network-changed-246ff4b3-1dd3-440f-aafb-c044dc9cb709 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 963.991157] env[62736]: DEBUG nova.compute.manager [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Refreshing instance network info cache due to event network-changed-246ff4b3-1dd3-440f-aafb-c044dc9cb709. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 963.991297] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] Acquiring lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 963.991439] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] Acquired lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 963.991683] env[62736]: DEBUG nova.network.neutron [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Refreshing network info cache for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 963.991992] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] Expecting reply to msg f658be1f671b401c8bf1bef4a366c62b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 963.999417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f658be1f671b401c8bf1bef4a366c62b [ 964.245562] env[62736]: ERROR nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 964.245562] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.245562] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 964.245562] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 964.245562] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.245562] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.245562] env[62736]: ERROR nova.compute.manager raise self.value [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 964.245562] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 964.245562] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.245562] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 964.246171] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.246171] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 964.246171] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 964.246171] env[62736]: ERROR nova.compute.manager [ 964.246171] env[62736]: Traceback (most recent call last): [ 964.246171] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 964.246171] env[62736]: listener.cb(fileno) [ 964.246171] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 964.246171] env[62736]: result = function(*args, **kwargs) [ 964.246171] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 964.246171] env[62736]: return func(*args, **kwargs) [ 964.246171] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 964.246171] env[62736]: raise e [ 964.246171] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.246171] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 964.246171] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 964.246171] env[62736]: created_port_ids = self._update_ports_for_instance( [ 964.246171] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 964.246171] env[62736]: with excutils.save_and_reraise_exception(): [ 964.246171] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.246171] env[62736]: self.force_reraise() [ 964.246171] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.246171] env[62736]: raise self.value [ 964.246171] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 964.246171] env[62736]: updated_port = self._update_port( [ 964.246171] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.246171] env[62736]: _ensure_no_port_binding_failure(port) [ 964.246171] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.246171] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 964.247148] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 964.247148] env[62736]: Removing descriptor: 17 [ 964.296688] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 964.321804] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 964.322083] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 964.322234] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 964.322408] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 964.322555] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 964.322796] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 964.322911] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 964.323086] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 964.323279] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 964.323400] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 964.323597] env[62736]: DEBUG nova.virt.hardware [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 964.324596] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa7fe378-e1d5-4319-9e6b-ccad0dee2725 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 964.332726] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d0c8807-ecba-4f15-813a-7607fd3ef8ac {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 964.347283] env[62736]: ERROR nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Traceback (most recent call last): [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] yield resources [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.driver.spawn(context, instance, image_meta, [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] vm_ref = self.build_virtual_machine(instance, [ 964.347283] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] for vif in network_info: [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return self._sync_wrapper(fn, *args, **kwargs) [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.wait() [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self[:] = self._gt.wait() [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return self._exit_event.wait() [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 964.347647] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] current.throw(*self._exc) [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] result = function(*args, **kwargs) [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return func(*args, **kwargs) [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise e [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] nwinfo = self.network_api.allocate_for_instance( [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] created_port_ids = self._update_ports_for_instance( [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] with excutils.save_and_reraise_exception(): [ 964.348054] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.force_reraise() [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise self.value [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] updated_port = self._update_port( [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] _ensure_no_port_binding_failure(port) [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise exception.PortBindingFailed(port_id=port['id']) [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 964.348439] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] [ 964.348439] env[62736]: INFO nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Terminating instance [ 964.352509] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 964.420812] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.155s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 964.421392] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 964.423083] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 54d77ddb68274852882b3cec12bf3628 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 964.455250] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 54d77ddb68274852882b3cec12bf3628 [ 964.512916] env[62736]: DEBUG nova.network.neutron [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 964.654425] env[62736]: DEBUG nova.network.neutron [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 964.655076] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] Expecting reply to msg 5bdb062d87ae4dfb8323e56d8b9c2925 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 964.663555] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bdb062d87ae4dfb8323e56d8b9c2925 [ 964.926833] env[62736]: DEBUG nova.compute.utils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 964.927884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg f57d1138241945148e1e11a63e590b8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 964.928816] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 964.928988] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 964.944200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f57d1138241945148e1e11a63e590b8b [ 964.980262] env[62736]: DEBUG nova.policy [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ea7e09048589421e96a4d19f8e884c8a', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '204d8f668a894827994c416d74f34c18', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 965.157475] env[62736]: DEBUG oslo_concurrency.lockutils [req-1d91f012-5a59-494d-b100-0b8e11ac3a83 req-0ba2986e-c67e-473f-99f4-21afc5c190aa service nova] Releasing lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 965.157940] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 965.158271] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 965.158582] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 0985fa80326d4b2fb356d0a9b1e26000 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.170180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0985fa80326d4b2fb356d0a9b1e26000 [ 965.222708] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "b3a23138-ab20-4c59-9d67-761afd7d4a06" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 965.222951] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "b3a23138-ab20-4c59-9d67-761afd7d4a06" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 965.223437] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 731493cd4cbb40c580807b447ce399d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.234667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 731493cd4cbb40c580807b447ce399d0 [ 965.272481] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Successfully created port: 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 965.432231] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 965.433796] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg d1dd422d87fc4a16ab82b0a88b074829 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.484202] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d1dd422d87fc4a16ab82b0a88b074829 [ 965.534109] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquiring lock "4320d2de-3271-4284-a551-1761e6fcd71e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 965.534494] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "4320d2de-3271-4284-a551-1761e6fcd71e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 965.535012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 468354baed4844dd85266cadf2f1e311 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.546350] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 468354baed4844dd85266cadf2f1e311 [ 965.702156] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 965.725610] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 965.727381] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 99669b65b4a34440b466adcee05e30f3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.760031] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99669b65b4a34440b466adcee05e30f3 [ 965.789607] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 965.790093] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg dcd8638dc59147a5ae0ed4b0ab4d380e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.799114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dcd8638dc59147a5ae0ed4b0ab4d380e [ 965.939305] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 479833a7749f4c9881afd72b34477514 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 965.978200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 479833a7749f4c9881afd72b34477514 [ 966.015729] env[62736]: DEBUG nova.compute.manager [req-0b468e54-3804-44d5-aaee-90a5f27bfb98 req-17bd7ed7-f08d-4530-85c0-6dd505e9dad6 service nova] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Received event network-vif-deleted-246ff4b3-1dd3-440f-aafb-c044dc9cb709 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 966.037997] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 966.039947] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 11d8ef419bb14e8c846b016e65354950 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.088250] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11d8ef419bb14e8c846b016e65354950 [ 966.248299] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 966.248597] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 966.250154] env[62736]: INFO nova.compute.claims [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 966.251807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg ad1e969ba1fc412e93cbb2ced104427a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.292357] env[62736]: ERROR nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 966.292357] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.292357] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.292357] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.292357] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.292357] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.292357] env[62736]: ERROR nova.compute.manager raise self.value [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.292357] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 966.292357] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.292357] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 966.292922] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.292922] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 966.292922] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 966.292922] env[62736]: ERROR nova.compute.manager [ 966.292922] env[62736]: Traceback (most recent call last): [ 966.292922] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 966.292922] env[62736]: listener.cb(fileno) [ 966.292922] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.292922] env[62736]: result = function(*args, **kwargs) [ 966.292922] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 966.292922] env[62736]: return func(*args, **kwargs) [ 966.292922] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.292922] env[62736]: raise e [ 966.292922] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.292922] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 966.292922] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.292922] env[62736]: created_port_ids = self._update_ports_for_instance( [ 966.292922] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.292922] env[62736]: with excutils.save_and_reraise_exception(): [ 966.292922] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.292922] env[62736]: self.force_reraise() [ 966.292922] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.292922] env[62736]: raise self.value [ 966.292922] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.292922] env[62736]: updated_port = self._update_port( [ 966.292922] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.292922] env[62736]: _ensure_no_port_binding_failure(port) [ 966.292922] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.292922] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 966.293908] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 966.293908] env[62736]: Removing descriptor: 17 [ 966.296218] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ad1e969ba1fc412e93cbb2ced104427a [ 966.297942] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 966.298327] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 966.298572] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 966.299013] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b0158f07-0fd2-41f2-92e5-7b1a33d4c359 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.308824] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32335140-afbd-4d33-ada1-ae1e445a8806 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.328947] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fc58f8bf-849a-452a-a1cb-daec65f662a7 could not be found. [ 966.329104] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 966.329278] env[62736]: INFO nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Took 0.03 seconds to destroy the instance on the hypervisor. [ 966.329562] env[62736]: DEBUG oslo.service.loopingcall [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 966.329773] env[62736]: DEBUG nova.compute.manager [-] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 966.329863] env[62736]: DEBUG nova.network.neutron [-] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 966.344353] env[62736]: DEBUG nova.network.neutron [-] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 966.344832] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 51d03d2a88c344acb27a23b6271a8881 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.352343] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51d03d2a88c344acb27a23b6271a8881 [ 966.442480] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 966.467998] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 966.468250] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 966.468405] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 966.468584] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 966.468729] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 966.468923] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 966.469141] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 966.469297] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 966.469476] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 966.469642] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 966.469812] env[62736]: DEBUG nova.virt.hardware [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 966.470642] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90366c54-f6fc-42f9-84b4-faf45b990018 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.478092] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c71bc1a5-a41d-497a-8fd1-1aa1f183ced8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 966.491358] env[62736]: ERROR nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Traceback (most recent call last): [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] yield resources [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.driver.spawn(context, instance, image_meta, [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] vm_ref = self.build_virtual_machine(instance, [ 966.491358] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] vif_infos = vmwarevif.get_vif_info(self._session, [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] for vif in network_info: [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return self._sync_wrapper(fn, *args, **kwargs) [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.wait() [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self[:] = self._gt.wait() [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return self._exit_event.wait() [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 966.491861] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] current.throw(*self._exc) [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] result = function(*args, **kwargs) [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return func(*args, **kwargs) [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise e [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] nwinfo = self.network_api.allocate_for_instance( [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] created_port_ids = self._update_ports_for_instance( [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] with excutils.save_and_reraise_exception(): [ 966.492299] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.force_reraise() [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise self.value [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] updated_port = self._update_port( [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] _ensure_no_port_binding_failure(port) [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise exception.PortBindingFailed(port_id=port['id']) [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 966.492735] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] [ 966.492735] env[62736]: INFO nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Terminating instance [ 966.493913] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 966.494060] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 966.494218] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 966.494608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 724c4045b1e6407caf661e91cb80a13f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.501249] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 724c4045b1e6407caf661e91cb80a13f [ 966.565471] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 966.763608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 98b3c869254c4b6aa50f05cedf8f1761 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.771180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98b3c869254c4b6aa50f05cedf8f1761 [ 966.846889] env[62736]: DEBUG nova.network.neutron [-] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 966.847663] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ed5256f591c24beea1eb78916b26daf8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 966.856015] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed5256f591c24beea1eb78916b26daf8 [ 967.016994] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 967.106033] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 967.106560] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0f00989f1d0c44149e6d1d5ae8c0146d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 967.114708] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0f00989f1d0c44149e6d1d5ae8c0146d [ 967.322322] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f4778af-a111-41ee-aa88-9989cbad7cea {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.329924] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-92bc35c7-3d43-475a-b0c5-3bab996ac659 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.359926] env[62736]: INFO nova.compute.manager [-] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Took 1.03 seconds to deallocate network for instance. [ 967.362317] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-037d93a3-9655-4983-996a-e8e1d3357644 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.364926] env[62736]: DEBUG nova.compute.claims [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 967.365097] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 967.370024] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4149358a-6760-4752-ace8-0586f491c897 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.382700] env[62736]: DEBUG nova.compute.provider_tree [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 967.383156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg dc0b96d37f104095aea0b9f5412d83a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 967.389725] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc0b96d37f104095aea0b9f5412d83a0 [ 967.608878] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 967.609536] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 967.609821] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 967.610130] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0913ce9e-e9a1-4036-b869-59c53b1e9215 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.620007] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-667107d2-3283-407d-95a4-820c85dbbd42 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 967.640965] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance effae28c-cbe1-4656-9a92-9829d929e3d5 could not be found. [ 967.641183] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 967.641356] env[62736]: INFO nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Took 0.03 seconds to destroy the instance on the hypervisor. [ 967.641645] env[62736]: DEBUG oslo.service.loopingcall [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 967.641872] env[62736]: DEBUG nova.compute.manager [-] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 967.641959] env[62736]: DEBUG nova.network.neutron [-] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 967.655866] env[62736]: DEBUG nova.network.neutron [-] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 967.656330] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c981d7142c5b4dcc893c8b514ed9cb53 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 967.662756] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c981d7142c5b4dcc893c8b514ed9cb53 [ 967.885617] env[62736]: DEBUG nova.scheduler.client.report [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 967.889709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 5215dfc5e164499289a2645cef5a02f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 967.900255] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5215dfc5e164499289a2645cef5a02f1 [ 968.040733] env[62736]: DEBUG nova.compute.manager [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Received event network-changed-26fd6fc5-060f-497f-9d38-23f8d5ec7bb1 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 968.040927] env[62736]: DEBUG nova.compute.manager [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Refreshing instance network info cache due to event network-changed-26fd6fc5-060f-497f-9d38-23f8d5ec7bb1. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 968.041138] env[62736]: DEBUG oslo_concurrency.lockutils [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] Acquiring lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 968.041275] env[62736]: DEBUG oslo_concurrency.lockutils [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] Acquired lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 968.041428] env[62736]: DEBUG nova.network.neutron [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Refreshing network info cache for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 968.041860] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] Expecting reply to msg d364f87989974444923959ae4b47793b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.048336] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d364f87989974444923959ae4b47793b [ 968.158752] env[62736]: DEBUG nova.network.neutron [-] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.159324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 9ccc97e099944587a459aa3b4eb0b80e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.168505] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9ccc97e099944587a459aa3b4eb0b80e [ 968.392174] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.143s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 968.392696] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 968.394667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg dba5dada1b734b5286a4b483b7f3e206 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.395687] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.830s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 968.397154] env[62736]: INFO nova.compute.claims [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 968.398762] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 48e247cc7a834379b59abf1c348e585e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.426848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dba5dada1b734b5286a4b483b7f3e206 [ 968.435354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 48e247cc7a834379b59abf1c348e585e [ 968.559062] env[62736]: DEBUG nova.network.neutron [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 968.625549] env[62736]: DEBUG nova.network.neutron [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 968.626110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] Expecting reply to msg 0bcf953fd55b4580bde5abdb759f417b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.633878] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bcf953fd55b4580bde5abdb759f417b [ 968.661142] env[62736]: INFO nova.compute.manager [-] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Took 1.02 seconds to deallocate network for instance. [ 968.663555] env[62736]: DEBUG nova.compute.claims [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 968.663851] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 968.901348] env[62736]: DEBUG nova.compute.utils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 968.902015] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 44ad5133018f48a6b2162928c2e5c984 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.902922] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 968.903087] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 968.907628] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 1bd6d24c38d04992991d8ab2cfe833de in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 968.915703] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bd6d24c38d04992991d8ab2cfe833de [ 968.916091] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 44ad5133018f48a6b2162928c2e5c984 [ 968.943444] env[62736]: DEBUG nova.policy [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b785ade05de40c88f4bf3ef92e5a25f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '721d9e362be84a33aa7956177650d445', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 969.130842] env[62736]: DEBUG oslo_concurrency.lockutils [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] Releasing lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 969.131121] env[62736]: DEBUG nova.compute.manager [req-52fba428-21cd-45e9-af96-ef81a6d14090 req-80aff9cd-5618-4fcf-99bf-e6ddd69f8b07 service nova] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Received event network-vif-deleted-26fd6fc5-060f-497f-9d38-23f8d5ec7bb1 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 969.195854] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Successfully created port: aba92c72-ee92-4337-b5d7-c7ad37a2aae4 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 969.405861] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 969.407554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 114fc25289504183a98bb17cce07280c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 969.444121] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 114fc25289504183a98bb17cce07280c [ 969.472781] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3e27391-8804-4985-9326-bb72caef2168 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.480707] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9b10752d-4352-4cf1-8b04-eb254183d7b3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.511478] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba5b1016-461f-44d0-b2a8-1afe1b289268 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.519287] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce0823ce-7a0d-4cf5-8046-6930ffc8328e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 969.532187] env[62736]: DEBUG nova.compute.provider_tree [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 969.532693] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 0e3a0dab41dd41b795b129bc40784721 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 969.539072] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e3a0dab41dd41b795b129bc40784721 [ 969.912482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg e387aaa01dd544738d063ca0795261d9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 969.944872] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e387aaa01dd544738d063ca0795261d9 [ 970.035359] env[62736]: DEBUG nova.scheduler.client.report [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 970.037705] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 023eed454a8c41f7b7ea699010a1c2dc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 970.049684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 023eed454a8c41f7b7ea699010a1c2dc [ 970.065349] env[62736]: DEBUG nova.compute.manager [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Received event network-changed-aba92c72-ee92-4337-b5d7-c7ad37a2aae4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 970.065537] env[62736]: DEBUG nova.compute.manager [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Refreshing instance network info cache due to event network-changed-aba92c72-ee92-4337-b5d7-c7ad37a2aae4. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 970.065748] env[62736]: DEBUG oslo_concurrency.lockutils [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] Acquiring lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.065883] env[62736]: DEBUG oslo_concurrency.lockutils [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] Acquired lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 970.066031] env[62736]: DEBUG nova.network.neutron [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Refreshing network info cache for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 970.066523] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] Expecting reply to msg c3bafb73cef44a2eb1c9569513b6fa20 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 970.072802] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3bafb73cef44a2eb1c9569513b6fa20 [ 970.082510] env[62736]: ERROR nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 970.082510] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.082510] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.082510] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.082510] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.082510] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.082510] env[62736]: ERROR nova.compute.manager raise self.value [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.082510] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 970.082510] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.082510] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 970.083051] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.083051] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 970.083051] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 970.083051] env[62736]: ERROR nova.compute.manager [ 970.083051] env[62736]: Traceback (most recent call last): [ 970.083051] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 970.083051] env[62736]: listener.cb(fileno) [ 970.083051] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.083051] env[62736]: result = function(*args, **kwargs) [ 970.083051] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 970.083051] env[62736]: return func(*args, **kwargs) [ 970.083051] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.083051] env[62736]: raise e [ 970.083051] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.083051] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 970.083051] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.083051] env[62736]: created_port_ids = self._update_ports_for_instance( [ 970.083051] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.083051] env[62736]: with excutils.save_and_reraise_exception(): [ 970.083051] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.083051] env[62736]: self.force_reraise() [ 970.083051] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.083051] env[62736]: raise self.value [ 970.083051] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.083051] env[62736]: updated_port = self._update_port( [ 970.083051] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.083051] env[62736]: _ensure_no_port_binding_failure(port) [ 970.083051] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.083051] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 970.083964] env[62736]: nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 970.083964] env[62736]: Removing descriptor: 17 [ 970.415983] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 970.440475] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 970.440861] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 970.441033] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 970.441216] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 970.441359] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 970.441503] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 970.441706] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 970.441891] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 970.442062] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 970.442221] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 970.442387] env[62736]: DEBUG nova.virt.hardware [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 970.443217] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cf1f856-af1e-48df-829a-929bc913c444 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.451008] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4cabd06-d35c-4aec-856a-6d9ea2f5901e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 970.463861] env[62736]: ERROR nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Traceback (most recent call last): [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] yield resources [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.driver.spawn(context, instance, image_meta, [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self._vmops.spawn(context, instance, image_meta, injected_files, [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] vm_ref = self.build_virtual_machine(instance, [ 970.463861] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] vif_infos = vmwarevif.get_vif_info(self._session, [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] for vif in network_info: [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return self._sync_wrapper(fn, *args, **kwargs) [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.wait() [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self[:] = self._gt.wait() [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return self._exit_event.wait() [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 970.464318] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] current.throw(*self._exc) [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] result = function(*args, **kwargs) [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return func(*args, **kwargs) [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise e [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] nwinfo = self.network_api.allocate_for_instance( [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] created_port_ids = self._update_ports_for_instance( [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] with excutils.save_and_reraise_exception(): [ 970.464770] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.force_reraise() [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise self.value [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] updated_port = self._update_port( [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] _ensure_no_port_binding_failure(port) [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise exception.PortBindingFailed(port_id=port['id']) [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 970.465230] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] [ 970.465230] env[62736]: INFO nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Terminating instance [ 970.465968] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 970.540039] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.144s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 970.540620] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 970.542389] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 1abe640f9a9c4851b403389955f81e2e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 970.543361] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 3.178s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 970.545018] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 12780be182d2441289d7c1cc338aca4c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 970.572882] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1abe640f9a9c4851b403389955f81e2e [ 970.576463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12780be182d2441289d7c1cc338aca4c [ 970.583629] env[62736]: DEBUG nova.network.neutron [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 970.651528] env[62736]: DEBUG nova.network.neutron [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 970.652123] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] Expecting reply to msg ea45ae5e1ddd404abcd3c1cfb8a0aa61 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 970.659284] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea45ae5e1ddd404abcd3c1cfb8a0aa61 [ 971.047971] env[62736]: DEBUG nova.compute.utils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 971.048702] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg acb861f47b824619bf99ac02f76eee8a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.053275] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 971.053685] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 971.062929] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acb861f47b824619bf99ac02f76eee8a [ 971.114982] env[62736]: DEBUG nova.policy [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ccb09ab46a4a45c3b361cd8a4129d471', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '32c9de32b740446c968d226b5926ecd8', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 971.118061] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dac7f1d-a19c-446d-b5ab-7bf029e29bba {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 971.125411] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-778b9558-9fb5-4ba8-b677-cd792c986db7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 971.154445] env[62736]: DEBUG oslo_concurrency.lockutils [req-645552be-1621-418e-9ed4-386779272d4a req-611267dc-7907-4cdb-9443-24f5fa823de6 service nova] Releasing lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 971.154978] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 971.155151] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 971.155565] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 4de73a95f9f5482cba071a23f8747532 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.157430] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9927ba78-5de9-4a8f-97fc-e46fd25f2175 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 971.161811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4de73a95f9f5482cba071a23f8747532 [ 971.166627] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aef5ccea-6e2e-47fb-9b6a-1e05fbc0c0f0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 971.180019] env[62736]: DEBUG nova.compute.provider_tree [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 971.180471] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c2707ce7d3bf4a4f9a2abbd584af7168 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.186553] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2707ce7d3bf4a4f9a2abbd584af7168 [ 971.420930] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Successfully created port: 4181d88c-19ef-422c-95b4-591cd4136c55 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 971.554263] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 971.556097] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 8efa2499fdc04bdb989660be5ad2c1ee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.590430] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8efa2499fdc04bdb989660be5ad2c1ee [ 971.675633] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 971.683199] env[62736]: DEBUG nova.scheduler.client.report [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 971.685801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c3a23758a2b1471b9fed0a5baf192e44 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.696160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c3a23758a2b1471b9fed0a5baf192e44 [ 971.781576] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 971.782125] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg dbf8a6cf91d2491682059c9e987e9f74 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.791192] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dbf8a6cf91d2491682059c9e987e9f74 [ 971.985749] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6a48c89a920e444daa37851826a03c0a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 971.994841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6a48c89a920e444daa37851826a03c0a [ 972.060776] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 2da640bdb64741b39ab4b54a9593c23a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.096981] env[62736]: DEBUG nova.compute.manager [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Received event network-vif-deleted-aba92c72-ee92-4337-b5d7-c7ad37a2aae4 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 972.096981] env[62736]: DEBUG nova.compute.manager [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Received event network-changed-4181d88c-19ef-422c-95b4-591cd4136c55 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 972.096981] env[62736]: DEBUG nova.compute.manager [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Refreshing instance network info cache due to event network-changed-4181d88c-19ef-422c-95b4-591cd4136c55. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 972.097173] env[62736]: DEBUG oslo_concurrency.lockutils [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] Acquiring lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 972.097312] env[62736]: DEBUG oslo_concurrency.lockutils [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] Acquired lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 972.097471] env[62736]: DEBUG nova.network.neutron [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Refreshing network info cache for port 4181d88c-19ef-422c-95b4-591cd4136c55 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 972.097881] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] Expecting reply to msg 625263daa1a443888e19713ccca3cfb3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.103903] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 625263daa1a443888e19713ccca3cfb3 [ 972.107361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2da640bdb64741b39ab4b54a9593c23a [ 972.188297] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.645s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 972.188886] env[62736]: ERROR nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Traceback (most recent call last): [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.driver.spawn(context, instance, image_meta, [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] vm_ref = self.build_virtual_machine(instance, [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] vif_infos = vmwarevif.get_vif_info(self._session, [ 972.188886] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] for vif in network_info: [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return self._sync_wrapper(fn, *args, **kwargs) [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.wait() [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self[:] = self._gt.wait() [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return self._exit_event.wait() [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] current.throw(*self._exc) [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 972.189303] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] result = function(*args, **kwargs) [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] return func(*args, **kwargs) [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise e [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] nwinfo = self.network_api.allocate_for_instance( [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] created_port_ids = self._update_ports_for_instance( [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] with excutils.save_and_reraise_exception(): [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] self.force_reraise() [ 972.189709] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise self.value [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] updated_port = self._update_port( [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] _ensure_no_port_binding_failure(port) [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] raise exception.PortBindingFailed(port_id=port['id']) [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] nova.exception.PortBindingFailed: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. [ 972.190092] env[62736]: ERROR nova.compute.manager [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] [ 972.190092] env[62736]: DEBUG nova.compute.utils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 972.190743] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 3.527s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 972.192539] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg e662064bf78f43ec8ff2396440659b6c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.193784] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Build of instance fc58f8bf-849a-452a-a1cb-daec65f662a7 was re-scheduled: Binding failed for port 246ff4b3-1dd3-440f-aafb-c044dc9cb709, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 972.194179] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 972.194388] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 972.194529] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 972.194678] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 972.195012] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a192e352434c44cf9901b67ea592de5a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.202478] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a192e352434c44cf9901b67ea592de5a [ 972.233302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e662064bf78f43ec8ff2396440659b6c [ 972.266000] env[62736]: ERROR nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 972.266000] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.266000] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 972.266000] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 972.266000] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.266000] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.266000] env[62736]: ERROR nova.compute.manager raise self.value [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 972.266000] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 972.266000] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.266000] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 972.266577] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.266577] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 972.266577] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 972.266577] env[62736]: ERROR nova.compute.manager [ 972.266577] env[62736]: Traceback (most recent call last): [ 972.266577] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 972.266577] env[62736]: listener.cb(fileno) [ 972.266577] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 972.266577] env[62736]: result = function(*args, **kwargs) [ 972.266577] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 972.266577] env[62736]: return func(*args, **kwargs) [ 972.266577] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 972.266577] env[62736]: raise e [ 972.266577] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.266577] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 972.266577] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 972.266577] env[62736]: created_port_ids = self._update_ports_for_instance( [ 972.266577] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 972.266577] env[62736]: with excutils.save_and_reraise_exception(): [ 972.266577] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.266577] env[62736]: self.force_reraise() [ 972.266577] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.266577] env[62736]: raise self.value [ 972.266577] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 972.266577] env[62736]: updated_port = self._update_port( [ 972.266577] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.266577] env[62736]: _ensure_no_port_binding_failure(port) [ 972.266577] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.266577] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 972.267735] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 972.267735] env[62736]: Removing descriptor: 17 [ 972.284101] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 972.284524] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 972.284709] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 972.284993] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bd5f168a-2ab7-470e-9e37-6b5f88eeb137 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.293680] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dff55d5d-3951-4149-9075-487f14ca726c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.314793] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b3a23138-ab20-4c59-9d67-761afd7d4a06 could not be found. [ 972.315023] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 972.315197] env[62736]: INFO nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Took 0.03 seconds to destroy the instance on the hypervisor. [ 972.315436] env[62736]: DEBUG oslo.service.loopingcall [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 972.315654] env[62736]: DEBUG nova.compute.manager [-] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 972.315744] env[62736]: DEBUG nova.network.neutron [-] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 972.329200] env[62736]: DEBUG nova.network.neutron [-] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.329833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a6323497f4fa432ea35d855d4c29345c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.336273] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6323497f4fa432ea35d855d4c29345c [ 972.564761] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 972.589794] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 972.590055] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 972.590210] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 972.590386] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 972.590528] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 972.590672] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 972.590874] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 972.591028] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 972.591189] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 972.591382] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 972.591504] env[62736]: DEBUG nova.virt.hardware [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 972.592394] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9a0f84a-83d3-460b-b7cf-4d72bb038ece {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.602258] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f057afa-54b5-45f2-abf8-e3320447620a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.616339] env[62736]: ERROR nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Traceback (most recent call last): [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] yield resources [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.driver.spawn(context, instance, image_meta, [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] vm_ref = self.build_virtual_machine(instance, [ 972.616339] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] vif_infos = vmwarevif.get_vif_info(self._session, [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] for vif in network_info: [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return self._sync_wrapper(fn, *args, **kwargs) [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.wait() [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self[:] = self._gt.wait() [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return self._exit_event.wait() [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 972.616819] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] current.throw(*self._exc) [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] result = function(*args, **kwargs) [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return func(*args, **kwargs) [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise e [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] nwinfo = self.network_api.allocate_for_instance( [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] created_port_ids = self._update_ports_for_instance( [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] with excutils.save_and_reraise_exception(): [ 972.617259] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.force_reraise() [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise self.value [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] updated_port = self._update_port( [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] _ensure_no_port_binding_failure(port) [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise exception.PortBindingFailed(port_id=port['id']) [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 972.617720] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] [ 972.617720] env[62736]: INFO nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Terminating instance [ 972.618506] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquiring lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 972.625003] env[62736]: DEBUG nova.network.neutron [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.704961] env[62736]: DEBUG nova.network.neutron [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.705473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] Expecting reply to msg 2b733d8297c94efba8c7aeeabf6601be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.714729] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2b733d8297c94efba8c7aeeabf6601be [ 972.719781] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 972.759295] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d23fac7-6dc3-4234-afee-a101fc19eb4d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.766467] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bc5ab04-3e1e-4a0d-bd44-1f380bdfa8ab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.796502] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.796975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 7bd3ce0703bc4f18946945316caa523c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.798134] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6899bc5f-7074-4f80-ac72-a98d297c534f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.804798] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50a62592-7359-4f81-b80d-678772d476f8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 972.808804] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bd3ce0703bc4f18946945316caa523c [ 972.818455] env[62736]: DEBUG nova.compute.provider_tree [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 972.818896] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0e7a7837bfc84d1888c6fa09978e68fd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.825595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e7a7837bfc84d1888c6fa09978e68fd [ 972.831612] env[62736]: DEBUG nova.network.neutron [-] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 972.831986] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 5d5313736505490ea870dc7728a206b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 972.838817] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d5313736505490ea870dc7728a206b7 [ 973.133677] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 973.133968] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 973.134554] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg a35eb8caa83547b983b8b7868bb8d445 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.144171] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a35eb8caa83547b983b8b7868bb8d445 [ 973.207911] env[62736]: DEBUG oslo_concurrency.lockutils [req-9bf395e6-a5ae-4387-9e56-6e2284af5cc7 req-f05131c4-d219-4b9e-987b-7f115508215d service nova] Releasing lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 973.208414] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquired lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 973.208596] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 973.209008] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 90b051cf0a7f4278af7b0584e9d1fd69 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.215445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90b051cf0a7f4278af7b0584e9d1fd69 [ 973.301065] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-fc58f8bf-849a-452a-a1cb-daec65f662a7" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 973.301287] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 973.301468] env[62736]: DEBUG nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 973.301636] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 973.315103] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 973.315595] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 492f19574de64240bc417242514ae9e5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.321392] env[62736]: DEBUG nova.scheduler.client.report [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 973.323770] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 0bc7ff6b4f8d4be5aac735b077196641 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.325545] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 492f19574de64240bc417242514ae9e5 [ 973.333618] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bc7ff6b4f8d4be5aac735b077196641 [ 973.334061] env[62736]: INFO nova.compute.manager [-] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Took 1.02 seconds to deallocate network for instance. [ 973.336252] env[62736]: DEBUG nova.compute.claims [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 973.336424] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.638968] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 973.639128] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 973.639249] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 973.639801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 8988817491d349049d0742033ba6508e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.650077] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8988817491d349049d0742033ba6508e [ 973.725770] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 973.800737] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.801232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 014e609014024dc8aa28e0b8cdb61562 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.809200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 014e609014024dc8aa28e0b8cdb61562 [ 973.817645] env[62736]: DEBUG nova.network.neutron [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 973.818047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 509e7f0fe72e4b38a4348f744850ddd2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.825134] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 509e7f0fe72e4b38a4348f744850ddd2 [ 973.826911] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.636s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.827538] env[62736]: ERROR nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Traceback (most recent call last): [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.driver.spawn(context, instance, image_meta, [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] vm_ref = self.build_virtual_machine(instance, [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] vif_infos = vmwarevif.get_vif_info(self._session, [ 973.827538] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] for vif in network_info: [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return self._sync_wrapper(fn, *args, **kwargs) [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.wait() [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self[:] = self._gt.wait() [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return self._exit_event.wait() [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] current.throw(*self._exc) [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 973.827927] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] result = function(*args, **kwargs) [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] return func(*args, **kwargs) [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise e [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] nwinfo = self.network_api.allocate_for_instance( [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] created_port_ids = self._update_ports_for_instance( [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] with excutils.save_and_reraise_exception(): [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] self.force_reraise() [ 973.828356] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise self.value [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] updated_port = self._update_port( [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] _ensure_no_port_binding_failure(port) [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] raise exception.PortBindingFailed(port_id=port['id']) [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] nova.exception.PortBindingFailed: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. [ 973.828779] env[62736]: ERROR nova.compute.manager [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] [ 973.828779] env[62736]: DEBUG nova.compute.utils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 973.829526] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Build of instance effae28c-cbe1-4656-9a92-9829d929e3d5 was re-scheduled: Binding failed for port 26fd6fc5-060f-497f-9d38-23f8d5ec7bb1, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 973.829942] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 973.830165] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquiring lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 973.830305] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Acquired lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 973.830474] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 973.830828] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 6f11cff18fc543448268a360e83c86e0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.832041] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.496s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.834036] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg b897c728ed5e4f44ac76f857fcd05954 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 973.835792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f11cff18fc543448268a360e83c86e0 [ 973.865833] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b897c728ed5e4f44ac76f857fcd05954 [ 974.123472] env[62736]: DEBUG nova.compute.manager [req-8d93e670-d26c-413d-99f5-e27815fce0e6 req-ba657304-7662-489b-ac08-01ad86fd33fb service nova] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Received event network-vif-deleted-4181d88c-19ef-422c-95b4-591cd4136c55 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 974.142316] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 974.142616] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Skipping network cache update for instance because it is Building. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9939}} [ 974.142616] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 974.142772] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.142929] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.143075] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.143221] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.143361] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.143505] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.143631] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 974.143771] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 974.144176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg c37344bd21aa4d569882cb86e835e930 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.153133] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c37344bd21aa4d569882cb86e835e930 [ 974.303491] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Releasing lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 974.304031] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 974.304240] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 974.304527] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5128b951-9a5b-4f8a-9e75-38e6e3a0e6f2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.313880] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7dd3a130-8111-4920-8c18-7002934a1940 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.325120] env[62736]: INFO nova.compute.manager [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: fc58f8bf-849a-452a-a1cb-daec65f662a7] Took 1.02 seconds to deallocate network for instance. [ 974.326707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b9c7cfdbdd164cd59ca5032f655a752c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.340985] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 4320d2de-3271-4284-a551-1761e6fcd71e could not be found. [ 974.341176] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 974.341343] env[62736]: INFO nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 974.341573] env[62736]: DEBUG oslo.service.loopingcall [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 974.341996] env[62736]: DEBUG nova.compute.manager [-] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 974.342089] env[62736]: DEBUG nova.network.neutron [-] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 974.348183] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 974.357463] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b9c7cfdbdd164cd59ca5032f655a752c [ 974.366325] env[62736]: DEBUG nova.network.neutron [-] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 974.366715] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6f478a05f35c4f909cc6e3a7749aa08b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.376651] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f478a05f35c4f909cc6e3a7749aa08b [ 974.396847] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0f52595-a679-4083-a225-d1c4891df92d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.404026] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3cd7e3f5-cd9a-4069-b127-b38d420a2aeb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.434644] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa1ecef6-f96f-4bc3-8b1d-af3706c64824 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.441394] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 974.442011] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 03bbd1e1792e4c3cbf21a73da3d64a56 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.443821] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2673e315-9101-4447-9740-a6a812281962 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 974.457714] env[62736]: DEBUG nova.compute.provider_tree [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 974.458314] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 8c668b95c6f445c6843a3fea9d64beee in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.460380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03bbd1e1792e4c3cbf21a73da3d64a56 [ 974.466200] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c668b95c6f445c6843a3fea9d64beee [ 974.646304] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.831763] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d25e40f3f26048d292fe8f47b8a480ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.860238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d25e40f3f26048d292fe8f47b8a480ab [ 974.870751] env[62736]: DEBUG nova.network.neutron [-] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 974.871114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg c173ddfc514b47ec961285283261dd72 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.878732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c173ddfc514b47ec961285283261dd72 [ 974.949925] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Releasing lock "refresh_cache-effae28c-cbe1-4656-9a92-9829d929e3d5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 974.950137] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 974.950311] env[62736]: DEBUG nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 974.950467] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 974.962166] env[62736]: DEBUG nova.scheduler.client.report [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 974.964347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 9a03990c3e3141998d686d67465aaac6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.966222] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 974.966700] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg a022af3177ce4d2e830c51894daf8e1c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 974.973050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a022af3177ce4d2e830c51894daf8e1c [ 974.974729] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a03990c3e3141998d686d67465aaac6 [ 975.847996] env[62736]: INFO nova.compute.manager [-] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Took 1.51 seconds to deallocate network for instance. [ 975.848617] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 2.017s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.849165] env[62736]: ERROR nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Traceback (most recent call last): [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.driver.spawn(context, instance, image_meta, [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self._vmops.spawn(context, instance, image_meta, injected_files, [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] vm_ref = self.build_virtual_machine(instance, [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] vif_infos = vmwarevif.get_vif_info(self._session, [ 975.849165] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] for vif in network_info: [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return self._sync_wrapper(fn, *args, **kwargs) [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.wait() [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self[:] = self._gt.wait() [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return self._exit_event.wait() [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] current.throw(*self._exc) [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 975.849596] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] result = function(*args, **kwargs) [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] return func(*args, **kwargs) [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise e [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] nwinfo = self.network_api.allocate_for_instance( [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] created_port_ids = self._update_ports_for_instance( [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] with excutils.save_and_reraise_exception(): [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] self.force_reraise() [ 975.850005] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise self.value [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] updated_port = self._update_port( [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] _ensure_no_port_binding_failure(port) [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] raise exception.PortBindingFailed(port_id=port['id']) [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] nova.exception.PortBindingFailed: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. [ 975.850442] env[62736]: ERROR nova.compute.manager [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] [ 975.850442] env[62736]: DEBUG nova.compute.utils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 975.851273] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Build of instance b3a23138-ab20-4c59-9d67-761afd7d4a06 was re-scheduled: Binding failed for port aba92c72-ee92-4337-b5d7-c7ad37a2aae4, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 975.851664] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 975.851899] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 975.852061] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 975.852216] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 975.852591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 91562cb425d44ab5b18d76668269158a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 975.853679] env[62736]: DEBUG nova.network.neutron [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 975.854153] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 18cf4ae43ca74b74976282c1b4ef7faf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 975.856669] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 1.210s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.856765] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.856877] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 975.858093] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcc741f9-ec21-4d99-8b91-346ca52031cd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.861903] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 91562cb425d44ab5b18d76668269158a [ 975.862423] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18cf4ae43ca74b74976282c1b4ef7faf [ 975.863346] env[62736]: DEBUG nova.compute.claims [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 975.863509] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.863709] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.865497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 4decf8fdbadf40d3bd8e645fe13bafa1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 975.872084] env[62736]: INFO nova.scheduler.client.report [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance fc58f8bf-849a-452a-a1cb-daec65f662a7 [ 975.877361] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 739ed6b7b7e8427b81d3439f44e6771a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 975.879653] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9ecc547-b818-4e61-abc6-135482e6a005 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.893542] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-31b51932-a314-47f9-8547-7848591f6990 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.902215] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4decf8fdbadf40d3bd8e645fe13bafa1 [ 975.902743] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 739ed6b7b7e8427b81d3439f44e6771a [ 975.904065] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-87afed14-ac30-4480-9da9-b44e2a2586d1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 975.933572] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181702MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 975.933726] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.357340] env[62736]: INFO nova.compute.manager [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] [instance: effae28c-cbe1-4656-9a92-9829d929e3d5] Took 1.41 seconds to deallocate network for instance. [ 976.359085] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 8d2b528a0d7940c38c0854620e2bb528 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.378849] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f7ed7e0c-eadb-49c2-92d6-9dc9012a071d tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "fc58f8bf-849a-452a-a1cb-daec65f662a7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.455s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.379911] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 976.394976] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8d2b528a0d7940c38c0854620e2bb528 [ 976.414672] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1df53cc-1793-4edf-a66b-27cf94997868 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.423803] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dbd453cb-2854-4a2e-9542-7294672cdd1b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.453615] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 976.454082] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 5090fa5094ca4fd5b22613743abef705 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.455239] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60104d89-e944-4ec7-b184-4b6d4cb5b148 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.461502] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5090fa5094ca4fd5b22613743abef705 [ 976.462757] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12031593-b05e-42c3-bb1e-a61d616609ee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 976.475667] env[62736]: DEBUG nova.compute.provider_tree [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 976.476152] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg e1f6e0db21b9444ab8572603bae32466 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.482855] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1f6e0db21b9444ab8572603bae32466 [ 976.863356] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 5ec765b15c0b4d188d3caa2693ebbe3c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.891394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5ec765b15c0b4d188d3caa2693ebbe3c [ 976.958297] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-b3a23138-ab20-4c59-9d67-761afd7d4a06" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 976.958551] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 976.958720] env[62736]: DEBUG nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 976.958880] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 976.973097] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 976.973659] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg d663aa62f7524c839c443339363c7fd5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.978509] env[62736]: DEBUG nova.scheduler.client.report [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 976.980739] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg bd4ea4133cf0447aa48540c74baa5397 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 976.981989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d663aa62f7524c839c443339363c7fd5 [ 976.991855] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bd4ea4133cf0447aa48540c74baa5397 [ 977.383183] env[62736]: INFO nova.scheduler.client.report [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Deleted allocations for instance effae28c-cbe1-4656-9a92-9829d929e3d5 [ 977.389841] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Expecting reply to msg 70a6acdc134247fbac096816211e822e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.417510] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 70a6acdc134247fbac096816211e822e [ 977.475394] env[62736]: DEBUG nova.network.neutron [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 977.475912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 00f04d51051f4783b2a0246399a485e7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.483843] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.620s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 977.484521] env[62736]: ERROR nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Traceback (most recent call last): [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.driver.spawn(context, instance, image_meta, [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] vm_ref = self.build_virtual_machine(instance, [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] vif_infos = vmwarevif.get_vif_info(self._session, [ 977.484521] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] for vif in network_info: [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return self._sync_wrapper(fn, *args, **kwargs) [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.wait() [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self[:] = self._gt.wait() [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return self._exit_event.wait() [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] current.throw(*self._exc) [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 977.484902] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] result = function(*args, **kwargs) [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] return func(*args, **kwargs) [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise e [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] nwinfo = self.network_api.allocate_for_instance( [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] created_port_ids = self._update_ports_for_instance( [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] with excutils.save_and_reraise_exception(): [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] self.force_reraise() [ 977.485309] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise self.value [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] updated_port = self._update_port( [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] _ensure_no_port_binding_failure(port) [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] raise exception.PortBindingFailed(port_id=port['id']) [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] nova.exception.PortBindingFailed: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. [ 977.485722] env[62736]: ERROR nova.compute.manager [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] [ 977.485722] env[62736]: DEBUG nova.compute.utils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 977.486924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00f04d51051f4783b2a0246399a485e7 [ 977.487266] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 1.554s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 977.488047] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg aee01a11b62d4c358468f6699d9804cb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.488863] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Build of instance 4320d2de-3271-4284-a551-1761e6fcd71e was re-scheduled: Binding failed for port 4181d88c-19ef-422c-95b4-591cd4136c55, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 977.489265] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 977.489510] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquiring lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 977.489660] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Acquired lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 977.489813] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 977.490179] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg bef15926e6eb4c1fb0e5fdd5af4b9915 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.496407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aee01a11b62d4c358468f6699d9804cb [ 977.501073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bef15926e6eb4c1fb0e5fdd5af4b9915 [ 977.892108] env[62736]: DEBUG oslo_concurrency.lockutils [None req-f72e5c39-6303-4d5c-9d38-770e18357219 tempest-ServersTestJSON-910337478 tempest-ServersTestJSON-910337478-project-member] Lock "effae28c-cbe1-4656-9a92-9829d929e3d5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 19.317s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 977.969899] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "9f629513-ffbd-4879-949e-816c160149b2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 977.970133] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "9f629513-ffbd-4879-949e-816c160149b2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 977.970645] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c510cb7223114e3d915425ff04512b45 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.977477] env[62736]: INFO nova.compute.manager [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: b3a23138-ab20-4c59-9d67-761afd7d4a06] Took 1.02 seconds to deallocate network for instance. [ 977.979039] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 8ba9fd482f9147028bca38e869129b61 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 977.979999] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c510cb7223114e3d915425ff04512b45 [ 977.991926] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg dfad5feef3054f61a24aa2f6edd7cf24 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.005417] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dfad5feef3054f61a24aa2f6edd7cf24 [ 978.019644] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ba9fd482f9147028bca38e869129b61 [ 978.171913] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 978.322228] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 978.323024] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 358a4b804f3f406d87296e891198661d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.332924] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 358a4b804f3f406d87296e891198661d [ 978.472750] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 978.474351] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg dc55ec82f2cf499f84bbdcc4445574a4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.483641] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 5a7964749b35431199071c264d9c079a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.508902] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dc55ec82f2cf499f84bbdcc4445574a4 [ 978.514352] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 5518542f5be84761ad70716068ff0db6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.520738] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a7964749b35431199071c264d9c079a [ 978.523283] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5518542f5be84761ad70716068ff0db6 [ 978.825908] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Releasing lock "refresh_cache-4320d2de-3271-4284-a551-1761e6fcd71e" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 978.826178] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 978.826368] env[62736]: DEBUG nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 978.826554] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 978.841247] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 978.841821] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 849daf8b37e84ac5863a41a87a29f065 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 978.849055] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 849daf8b37e84ac5863a41a87a29f065 [ 979.001025] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.007382] env[62736]: INFO nova.scheduler.client.report [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Deleted allocations for instance b3a23138-ab20-4c59-9d67-761afd7d4a06 [ 979.013884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 12f9c73c42274300a3047862c487c3a4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 979.017049] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance b3a23138-ab20-4c59-9d67-761afd7d4a06 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 979.017049] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 59aec7eaaec7418b98994f7a9f621968 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 979.027423] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12f9c73c42274300a3047862c487c3a4 [ 979.036378] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59aec7eaaec7418b98994f7a9f621968 [ 979.343824] env[62736]: DEBUG nova.network.neutron [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 979.344394] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg adafe699d10e49bdb95a542e63246376 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 979.353535] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adafe699d10e49bdb95a542e63246376 [ 979.515546] env[62736]: DEBUG oslo_concurrency.lockutils [None req-c0ddec34-58e3-4854-b62b-4b7db05135cd tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "b3a23138-ab20-4c59-9d67-761afd7d4a06" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 14.292s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 979.519419] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 4320d2de-3271-4284-a551-1761e6fcd71e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 979.520118] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 5d32845bbb9949868475af0f8a2c97b7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 979.537623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d32845bbb9949868475af0f8a2c97b7 [ 979.847267] env[62736]: INFO nova.compute.manager [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] [instance: 4320d2de-3271-4284-a551-1761e6fcd71e] Took 1.02 seconds to deallocate network for instance. [ 979.849113] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg d6cf2b6cbc3947e593059a7d48c7c969 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 979.955672] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d6cf2b6cbc3947e593059a7d48c7c969 [ 980.023324] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Instance 9f629513-ffbd-4879-949e-816c160149b2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=62736) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 980.023633] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 980.023683] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 980.064331] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acc67afb-6720-4d09-8487-a8edf3e2550d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.071659] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7a10fbb-43f7-4a3d-b788-065344b18115 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.101600] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c690e9d6-ebd5-41cb-8af4-24cd4fd99fcb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.108616] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57fcc732-c91c-44a1-9f6d-ea297a11ca6c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.122522] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 980.123004] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 57f8373459884200af44fcd084712cb5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.130863] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 57f8373459884200af44fcd084712cb5 [ 980.354783] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg e89aafe7f73e45949a8300d1392a203a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.387958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e89aafe7f73e45949a8300d1392a203a [ 980.625944] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 980.628178] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3497b5df1b2d4fe6b8a479cfbf520860 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.638913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3497b5df1b2d4fe6b8a479cfbf520860 [ 980.728337] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "267e30d2-c00c-4edf-8665-cfa88eba0938" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 980.728567] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.729042] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 5feb00c66c5145738c6f2760cbb8bdab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.741179] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5feb00c66c5145738c6f2760cbb8bdab [ 980.757986] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "8e17c8c1-e662-4695-9380-a43b7dd4ced6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 980.758215] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "8e17c8c1-e662-4695-9380-a43b7dd4ced6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.758895] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 4c5819454d184244b9c23a0c70ea2594 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.766485] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c5819454d184244b9c23a0c70ea2594 [ 980.875928] env[62736]: INFO nova.scheduler.client.report [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Deleted allocations for instance 4320d2de-3271-4284-a551-1761e6fcd71e [ 980.882979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Expecting reply to msg 98c26c9e412641149dd38ec127585c91 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 980.893140] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 98c26c9e412641149dd38ec127585c91 [ 981.131122] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 981.131469] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 3.644s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.131598] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 2.131s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 981.133018] env[62736]: INFO nova.compute.claims [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 981.134528] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 218446bfb6a34bd18a820496d0abd4d7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 981.164335] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 218446bfb6a34bd18a820496d0abd4d7 [ 981.231307] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 981.233112] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 62ff09106765455196acc711a0a3fdab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 981.261092] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 981.262740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 2c8f2212a0434794af107e7908d6dc6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 981.264094] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62ff09106765455196acc711a0a3fdab [ 981.290201] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2c8f2212a0434794af107e7908d6dc6a [ 981.384680] env[62736]: DEBUG oslo_concurrency.lockutils [None req-628cc1df-be75-407d-9f77-1fef73d44376 tempest-ServerActionsTestJSON-972253171 tempest-ServerActionsTestJSON-972253171-project-member] Lock "4320d2de-3271-4284-a551-1761e6fcd71e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 15.850s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.638114] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 9743e3ba56e445398b0ff0feed0ef446 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 981.647302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9743e3ba56e445398b0ff0feed0ef446 [ 981.749282] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 981.778732] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 982.189349] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-497ea7e2-6983-4c8d-8b81-82f3aed1ffab {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.196819] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e6f0d6d-b9a6-4a92-92aa-c6e432674fc1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.225112] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-542c7087-fba4-40e4-ac1b-941d8ef50989 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.231926] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39eec19e-b6ea-4533-bf35-57b03f9f5825 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.247040] env[62736]: DEBUG nova.compute.provider_tree [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 982.247557] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 3e9f839142e14b42814c75b44608752a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 982.254811] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3e9f839142e14b42814c75b44608752a [ 982.750746] env[62736]: DEBUG nova.scheduler.client.report [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 982.753149] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg cc0f34e5e29149af824c53dad1fe7eb1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 982.764766] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cc0f34e5e29149af824c53dad1fe7eb1 [ 983.256197] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.124s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 983.257514] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 983.258484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 31115abe2b0c453a906129bce51ad94e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 983.261368] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 1.512s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 983.262911] env[62736]: INFO nova.compute.claims [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 983.264503] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg ea73acda9f9442e5a44d2b0aa54f44b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 983.297132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 31115abe2b0c453a906129bce51ad94e [ 983.301343] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ea73acda9f9442e5a44d2b0aa54f44b9 [ 983.762869] env[62736]: DEBUG nova.compute.utils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 983.763530] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ae297e18fefc48e4aa5389bd802ab6c5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 983.764565] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 983.764742] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 983.767730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 60022c7fc1f14d6d97a82a0fe6555854 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 983.778493] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60022c7fc1f14d6d97a82a0fe6555854 [ 983.779068] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ae297e18fefc48e4aa5389bd802ab6c5 [ 983.815262] env[62736]: DEBUG nova.policy [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 984.069139] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Successfully created port: ced04c10-bd0f-4846-9e80-67425c845633 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 984.272997] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 984.274753] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 8f0927a4f8144d45a59baad17c019bcf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 984.324600] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f0927a4f8144d45a59baad17c019bcf [ 984.327197] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06bd8308-d3c0-44ce-b6db-5ea2e603000b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.334777] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37693d1b-76af-4000-b045-886374de9b5a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.366126] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-becf8e59-e151-48eb-9556-4c897c32455a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.373313] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55160e64-1057-48f4-9e13-1a476c5ca46a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.386776] env[62736]: DEBUG nova.compute.provider_tree [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 984.387243] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 615868db3e1c465d93cb60388168dbb5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 984.395740] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 615868db3e1c465d93cb60388168dbb5 [ 984.680759] env[62736]: DEBUG nova.compute.manager [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Received event network-changed-ced04c10-bd0f-4846-9e80-67425c845633 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 984.680967] env[62736]: DEBUG nova.compute.manager [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Refreshing instance network info cache due to event network-changed-ced04c10-bd0f-4846-9e80-67425c845633. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 984.681179] env[62736]: DEBUG oslo_concurrency.lockutils [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] Acquiring lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 984.681316] env[62736]: DEBUG oslo_concurrency.lockutils [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] Acquired lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 984.681469] env[62736]: DEBUG nova.network.neutron [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Refreshing network info cache for port ced04c10-bd0f-4846-9e80-67425c845633 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 984.681884] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] Expecting reply to msg c00fa14d376e406ea5b415007d746ad0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 984.689917] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c00fa14d376e406ea5b415007d746ad0 [ 984.779235] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 073dff759752488191f7ea957d520d78 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 984.810401] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 073dff759752488191f7ea957d520d78 [ 984.890414] env[62736]: DEBUG nova.scheduler.client.report [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 984.893546] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 86af82d54363497d88f28ea9487e2d50 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 984.896710] env[62736]: ERROR nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 984.896710] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 984.896710] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 984.896710] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 984.896710] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 984.896710] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 984.896710] env[62736]: ERROR nova.compute.manager raise self.value [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 984.896710] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 984.896710] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 984.896710] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 984.897202] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 984.897202] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 984.897202] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 984.897202] env[62736]: ERROR nova.compute.manager [ 984.897202] env[62736]: Traceback (most recent call last): [ 984.897202] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 984.897202] env[62736]: listener.cb(fileno) [ 984.897202] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 984.897202] env[62736]: result = function(*args, **kwargs) [ 984.897202] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 984.897202] env[62736]: return func(*args, **kwargs) [ 984.897202] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 984.897202] env[62736]: raise e [ 984.897202] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 984.897202] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 984.897202] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 984.897202] env[62736]: created_port_ids = self._update_ports_for_instance( [ 984.897202] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 984.897202] env[62736]: with excutils.save_and_reraise_exception(): [ 984.897202] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 984.897202] env[62736]: self.force_reraise() [ 984.897202] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 984.897202] env[62736]: raise self.value [ 984.897202] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 984.897202] env[62736]: updated_port = self._update_port( [ 984.897202] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 984.897202] env[62736]: _ensure_no_port_binding_failure(port) [ 984.897202] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 984.897202] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 984.898010] env[62736]: nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 984.898010] env[62736]: Removing descriptor: 19 [ 984.904443] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 86af82d54363497d88f28ea9487e2d50 [ 985.198385] env[62736]: DEBUG nova.network.neutron [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 985.274602] env[62736]: DEBUG nova.network.neutron [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 985.275119] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] Expecting reply to msg 380348a146234e5fba82fe8e6ea126d4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.282428] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 985.285040] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 380348a146234e5fba82fe8e6ea126d4 [ 985.306158] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 985.306499] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 985.306660] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 985.306839] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 985.306981] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 985.307122] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 985.307319] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 985.307472] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 985.307627] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 985.307782] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 985.307963] env[62736]: DEBUG nova.virt.hardware [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 985.308785] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aacda8c9-be5e-40cc-8777-3db065ce4c3b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.316711] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d8f960e-d62e-4189-b7bb-0ff558d1390d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 985.330738] env[62736]: ERROR nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] Traceback (most recent call last): [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] yield resources [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.driver.spawn(context, instance, image_meta, [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] vm_ref = self.build_virtual_machine(instance, [ 985.330738] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] vif_infos = vmwarevif.get_vif_info(self._session, [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] for vif in network_info: [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return self._sync_wrapper(fn, *args, **kwargs) [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.wait() [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self[:] = self._gt.wait() [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return self._exit_event.wait() [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 985.331353] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] current.throw(*self._exc) [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] result = function(*args, **kwargs) [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return func(*args, **kwargs) [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise e [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] nwinfo = self.network_api.allocate_for_instance( [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] created_port_ids = self._update_ports_for_instance( [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] with excutils.save_and_reraise_exception(): [ 985.332021] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.force_reraise() [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise self.value [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] updated_port = self._update_port( [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] _ensure_no_port_binding_failure(port) [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise exception.PortBindingFailed(port_id=port['id']) [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 985.332683] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] [ 985.332683] env[62736]: INFO nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Terminating instance [ 985.333259] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 985.396152] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.135s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 985.396684] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 985.398525] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 4b5323ece0ab4041b2705cc6d019065e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.399571] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 3.621s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 985.400951] env[62736]: INFO nova.compute.claims [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 985.402370] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 1bc0940b89a844aea1f73a1c53632e8b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.435237] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b5323ece0ab4041b2705cc6d019065e [ 985.440916] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1bc0940b89a844aea1f73a1c53632e8b [ 985.777944] env[62736]: DEBUG oslo_concurrency.lockutils [req-1ab1e94b-5ce5-45d1-81c5-0ec80aa4efb5 req-25308409-e0dc-41c7-974d-261b71581430 service nova] Releasing lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 985.778320] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 985.778504] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 985.778941] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg efa9ec5c104b48ad8a4a4dd226c2151d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.785945] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg efa9ec5c104b48ad8a4a4dd226c2151d [ 985.905281] env[62736]: DEBUG nova.compute.utils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 985.905892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg b3420a12e9d04d748ce34dfc88568374 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.906870] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Not allocating networking since 'none' was specified. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1977}} [ 985.912477] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 972efa58ba61431db11cebc40f7ba95a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 985.918139] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b3420a12e9d04d748ce34dfc88568374 [ 985.920529] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 972efa58ba61431db11cebc40f7ba95a [ 986.295703] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 986.359641] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.360154] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 6f864eec94064a8dad4a2133444542da in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 986.367960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6f864eec94064a8dad4a2133444542da [ 986.408145] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 986.409995] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 1ec37bb58b244a7181028bb75f5cf7fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 986.438531] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1ec37bb58b244a7181028bb75f5cf7fc [ 986.530212] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f69e4499-423c-46bb-8aad-b70653130027 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.537685] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed33f62d-34dd-4627-aabf-2a466f1ec4cc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.567650] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-156df82e-8692-4835-809c-5fc836fd9933 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.574952] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f7f5ddd-10ea-4701-9fa2-afd6c665da74 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.588227] env[62736]: DEBUG nova.compute.provider_tree [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 986.588722] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 4ee8bfd434c04076a53a79e6c2ea04f0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 986.596347] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4ee8bfd434c04076a53a79e6c2ea04f0 [ 986.706723] env[62736]: DEBUG nova.compute.manager [req-bb5907a3-4e4b-44a7-86db-58b8158e9871 req-daaa81f9-9c22-4e22-a688-79371c780c04 service nova] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Received event network-vif-deleted-ced04c10-bd0f-4846-9e80-67425c845633 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 986.862503] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 986.862892] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 986.863080] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 986.863352] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-83c26c19-445a-45ee-8d7a-aa05fd78d243 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.872649] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82e67e16-daed-445e-93a5-b037e227e6e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 986.893327] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9f629513-ffbd-4879-949e-816c160149b2 could not be found. [ 986.893521] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 986.893691] env[62736]: INFO nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Took 0.03 seconds to destroy the instance on the hypervisor. [ 986.893930] env[62736]: DEBUG oslo.service.loopingcall [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 986.894138] env[62736]: DEBUG nova.compute.manager [-] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 986.894226] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f629513-ffbd-4879-949e-816c160149b2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 986.908014] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 986.908462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 402296a3f6d44d95ac7d13cab5c5b8a1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 986.915005] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 83d7ae478dc142a2b58d1413b66c18df in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 986.915687] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 402296a3f6d44d95ac7d13cab5c5b8a1 [ 986.941598] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 83d7ae478dc142a2b58d1413b66c18df [ 987.091082] env[62736]: DEBUG nova.scheduler.client.report [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 987.093380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 458cc2693e304d62be471880ba8f2caf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 987.104930] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 458cc2693e304d62be471880ba8f2caf [ 987.413175] env[62736]: DEBUG nova.network.neutron [-] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 987.413653] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a3faa313444a479f942f86b288ccd5d6 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 987.417925] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 987.422975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a3faa313444a479f942f86b288ccd5d6 [ 987.439588] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 987.439826] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 987.439980] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 987.440214] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 987.440314] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 987.440453] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 987.440649] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 987.440803] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 987.440958] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 987.441113] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 987.441277] env[62736]: DEBUG nova.virt.hardware [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 987.442305] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6e3745b2-9208-44c6-bd5e-caf900f99886 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 987.449847] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07c21bf1-71f5-43ca-8ea6-c8db3d3077e2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 987.462712] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 987.468233] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Creating folder: Project (e2d2c93d1f1542978563392cc99f270d). Parent ref: group-v107913. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 987.468474] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-7feec439-f9ed-481a-88d1-899f631e6833 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 987.477141] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Created folder: Project (e2d2c93d1f1542978563392cc99f270d) in parent group-v107913. [ 987.477314] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Creating folder: Instances. Parent ref: group-v107942. {{(pid=62736) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 987.477513] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0858cc06-ed51-4d49-9bda-ec2172fd0b75 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 987.485392] env[62736]: INFO nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Created folder: Instances in parent group-v107942. [ 987.485600] env[62736]: DEBUG oslo.service.loopingcall [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 987.485774] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 987.485954] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-364e5cbd-b08e-4496-b607-04f5995c92e1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 987.502566] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 987.502566] env[62736]: value = "task-397787" [ 987.502566] env[62736]: _type = "Task" [ 987.502566] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 987.509364] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397787, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 987.596010] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.196s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 987.596575] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 987.598332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg a96823e9ee8842fe9b42a162212b99b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 987.629496] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a96823e9ee8842fe9b42a162212b99b4 [ 987.915489] env[62736]: INFO nova.compute.manager [-] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Took 1.02 seconds to deallocate network for instance. [ 987.917864] env[62736]: DEBUG nova.compute.claims [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 987.918047] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 987.918258] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 987.920046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d5d8cc82835345d385732d49b4f548fe in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 987.948461] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d5d8cc82835345d385732d49b4f548fe [ 988.012351] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397787, 'name': CreateVM_Task, 'duration_secs': 0.237424} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 988.012525] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 988.012921] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 988.013080] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 988.013384] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 988.013852] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-96afd77e-961c-4289-a5e8-af5af64dedcd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.018099] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 988.018099] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]520a93b8-2816-58b0-73f3-d6da54229aed" [ 988.018099] env[62736]: _type = "Task" [ 988.018099] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 988.025186] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]520a93b8-2816-58b0-73f3-d6da54229aed, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 988.101267] env[62736]: DEBUG nova.compute.utils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 988.101975] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 6094f942d2514b7d8f47a48056bb93fb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 988.103612] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 988.103832] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 988.111912] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6094f942d2514b7d8f47a48056bb93fb [ 988.144180] env[62736]: DEBUG nova.policy [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '5b785ade05de40c88f4bf3ef92e5a25f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '721d9e362be84a33aa7956177650d445', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 988.447000] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Successfully created port: 923b9398-72d1-454e-bc28-1e0bdbf893e1 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 988.471021] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5c69aa6-c401-48d9-ae13-cfee439eb4c1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.478936] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8ec6df5-ce1b-44f6-980f-e2264df143db {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.509455] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ef32363-f69e-4700-a66b-ced03a3f5fcd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.516319] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9f1e9eb-fe72-4d2b-ab0e-3165c48a5a15 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.531556] env[62736]: DEBUG nova.compute.provider_tree [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 988.532060] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 68fc7885b53a4294a75a9476c117d458 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 988.535971] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]520a93b8-2816-58b0-73f3-d6da54229aed, 'name': SearchDatastore_Task, 'duration_secs': 0.009356} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 988.536430] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 988.536646] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 988.536866] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 988.537004] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 988.537170] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 988.537389] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ae6eb0df-40a4-4fab-9d29-258cc807b3bd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.539157] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 68fc7885b53a4294a75a9476c117d458 [ 988.544950] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 988.545112] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 988.545756] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-80c6bf41-5254-4d9b-bde2-f3f477c0bec2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 988.551370] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 988.551370] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5293e27c-6801-7515-0369-6c24472f2346" [ 988.551370] env[62736]: _type = "Task" [ 988.551370] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 988.558308] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5293e27c-6801-7515-0369-6c24472f2346, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 988.604467] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 988.606123] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 6ee82c5726a445b08bf05955254f7548 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 988.634259] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ee82c5726a445b08bf05955254f7548 [ 989.038086] env[62736]: DEBUG nova.scheduler.client.report [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 989.040606] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 17b9e503eb6c443c9ba957efc395075d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 989.052349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 17b9e503eb6c443c9ba957efc395075d [ 989.064471] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5293e27c-6801-7515-0369-6c24472f2346, 'name': SearchDatastore_Task, 'duration_secs': 0.007202} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 989.065365] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-10ea26d6-6253-4c21-879f-cc696e034256 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.070759] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 989.070759] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]5250c702-ca2c-f417-6685-c1a75d262e88" [ 989.070759] env[62736]: _type = "Task" [ 989.070759] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 989.078088] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5250c702-ca2c-f417-6685-c1a75d262e88, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 989.110601] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 36826856def545b294b0b87064d7053b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 989.143406] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 36826856def545b294b0b87064d7053b [ 989.147142] env[62736]: DEBUG nova.compute.manager [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Received event network-changed-923b9398-72d1-454e-bc28-1e0bdbf893e1 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 989.147594] env[62736]: DEBUG nova.compute.manager [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Refreshing instance network info cache due to event network-changed-923b9398-72d1-454e-bc28-1e0bdbf893e1. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 989.147816] env[62736]: DEBUG oslo_concurrency.lockutils [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] Acquiring lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 989.147959] env[62736]: DEBUG oslo_concurrency.lockutils [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] Acquired lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 989.148132] env[62736]: DEBUG nova.network.neutron [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Refreshing network info cache for port 923b9398-72d1-454e-bc28-1e0bdbf893e1 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 989.148617] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] Expecting reply to msg a5da657a19304bafa74f16e08f038268 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 989.154808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a5da657a19304bafa74f16e08f038268 [ 989.297818] env[62736]: ERROR nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 989.297818] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 989.297818] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 989.297818] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 989.297818] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 989.297818] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 989.297818] env[62736]: ERROR nova.compute.manager raise self.value [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 989.297818] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 989.297818] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 989.297818] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 989.298378] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 989.298378] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 989.298378] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 989.298378] env[62736]: ERROR nova.compute.manager [ 989.298378] env[62736]: Traceback (most recent call last): [ 989.298378] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 989.298378] env[62736]: listener.cb(fileno) [ 989.298378] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 989.298378] env[62736]: result = function(*args, **kwargs) [ 989.298378] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 989.298378] env[62736]: return func(*args, **kwargs) [ 989.298378] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 989.298378] env[62736]: raise e [ 989.298378] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 989.298378] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 989.298378] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 989.298378] env[62736]: created_port_ids = self._update_ports_for_instance( [ 989.298378] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 989.298378] env[62736]: with excutils.save_and_reraise_exception(): [ 989.298378] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 989.298378] env[62736]: self.force_reraise() [ 989.298378] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 989.298378] env[62736]: raise self.value [ 989.298378] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 989.298378] env[62736]: updated_port = self._update_port( [ 989.298378] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 989.298378] env[62736]: _ensure_no_port_binding_failure(port) [ 989.298378] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 989.298378] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 989.299329] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 989.299329] env[62736]: Removing descriptor: 19 [ 989.543472] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.625s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 989.544283] env[62736]: ERROR nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] Traceback (most recent call last): [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.driver.spawn(context, instance, image_meta, [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] vm_ref = self.build_virtual_machine(instance, [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] vif_infos = vmwarevif.get_vif_info(self._session, [ 989.544283] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] for vif in network_info: [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return self._sync_wrapper(fn, *args, **kwargs) [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.wait() [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self[:] = self._gt.wait() [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return self._exit_event.wait() [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] current.throw(*self._exc) [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 989.544660] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] result = function(*args, **kwargs) [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] return func(*args, **kwargs) [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise e [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] nwinfo = self.network_api.allocate_for_instance( [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] created_port_ids = self._update_ports_for_instance( [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] with excutils.save_and_reraise_exception(): [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] self.force_reraise() [ 989.545044] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise self.value [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] updated_port = self._update_port( [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] _ensure_no_port_binding_failure(port) [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] raise exception.PortBindingFailed(port_id=port['id']) [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] nova.exception.PortBindingFailed: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. [ 989.545429] env[62736]: ERROR nova.compute.manager [instance: 9f629513-ffbd-4879-949e-816c160149b2] [ 989.545875] env[62736]: DEBUG nova.compute.utils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 989.547572] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Build of instance 9f629513-ffbd-4879-949e-816c160149b2 was re-scheduled: Binding failed for port ced04c10-bd0f-4846-9e80-67425c845633, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 989.548033] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 989.548344] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 989.548556] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 989.548773] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 989.549238] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ab8620d108474af2955f9f211fd54761 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 989.555804] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab8620d108474af2955f9f211fd54761 [ 989.581252] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]5250c702-ca2c-f417-6685-c1a75d262e88, 'name': SearchDatastore_Task, 'duration_secs': 0.008766} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 989.581543] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Releasing lock "[datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 989.581840] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Copying Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 989.582123] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-80ab2cba-e994-4980-ad86-c6d7af00002f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.589112] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 989.589112] env[62736]: value = "task-397788" [ 989.589112] env[62736]: _type = "Task" [ 989.589112] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 989.596741] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397788, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 989.613955] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 989.638189] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 989.638502] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 989.638714] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 989.638958] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 989.639146] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 989.639330] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 989.639614] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 989.639835] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 989.640088] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 989.640313] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 989.640542] env[62736]: DEBUG nova.virt.hardware [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 989.641551] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47e52068-ca88-4088-94b0-713597a058eb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.648524] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8caa9d6-18f0-49cd-ab63-77c4b9eea697 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 989.664371] env[62736]: ERROR nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Traceback (most recent call last): [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] yield resources [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.driver.spawn(context, instance, image_meta, [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] vm_ref = self.build_virtual_machine(instance, [ 989.664371] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] vif_infos = vmwarevif.get_vif_info(self._session, [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] for vif in network_info: [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return self._sync_wrapper(fn, *args, **kwargs) [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.wait() [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self[:] = self._gt.wait() [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return self._exit_event.wait() [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 989.665038] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] current.throw(*self._exc) [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] result = function(*args, **kwargs) [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return func(*args, **kwargs) [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise e [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] nwinfo = self.network_api.allocate_for_instance( [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] created_port_ids = self._update_ports_for_instance( [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] with excutils.save_and_reraise_exception(): [ 989.665665] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.force_reraise() [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise self.value [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] updated_port = self._update_port( [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] _ensure_no_port_binding_failure(port) [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise exception.PortBindingFailed(port_id=port['id']) [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 989.666293] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] [ 989.666293] env[62736]: INFO nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Terminating instance [ 989.666971] env[62736]: DEBUG nova.network.neutron [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 989.668848] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 989.737356] env[62736]: DEBUG nova.network.neutron [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 989.738050] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] Expecting reply to msg a4d4339269e34fdcbfdd389f1c1a98ea in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 989.748078] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a4d4339269e34fdcbfdd389f1c1a98ea [ 990.065757] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 990.098675] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397788, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.424364} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 990.101020] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Copied Virtual Disk [datastore1] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore1] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 990.101305] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 990.101593] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-011ec4c7-fa9e-4d4c-97db-46e0605083ce {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.107615] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 990.107615] env[62736]: value = "task-397789" [ 990.107615] env[62736]: _type = "Task" [ 990.107615] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 990.115350] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397789, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 990.136439] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 990.137006] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg db6a1a10e65e448f92450578f062d297 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 990.146298] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db6a1a10e65e448f92450578f062d297 [ 990.240617] env[62736]: DEBUG oslo_concurrency.lockutils [req-1b61d1ed-f64b-4544-a2cf-9f6b069b9cfb req-b1daa767-c8fd-4692-87fb-5ba2744dabbd service nova] Releasing lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 990.241082] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 990.241323] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 990.241801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 49625ec9c4004a1abef07aa794127119 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 990.248340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49625ec9c4004a1abef07aa794127119 [ 990.617798] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397789, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.064149} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 990.618172] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 990.619032] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f7f026b-61ea-45b9-967b-102d1cabf73c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.638655] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Reconfiguring VM instance instance-00000062 to attach disk [datastore1] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 990.639240] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-9f629513-ffbd-4879-949e-816c160149b2" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 990.639536] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 990.639786] env[62736]: DEBUG nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 990.640024] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 990.641735] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-98762ddb-2824-41fd-8027-a0685a88d0c2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 990.660825] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 990.660825] env[62736]: value = "task-397790" [ 990.660825] env[62736]: _type = "Task" [ 990.660825] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 990.669031] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397790, 'name': ReconfigVM_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 990.669878] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 990.670491] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg d0626719653946439facff8295572cb4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 990.677850] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0626719653946439facff8295572cb4 [ 990.758694] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 990.825189] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 990.825719] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg b6d3275173684f738c7c96a38d126110 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 990.834192] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b6d3275173684f738c7c96a38d126110 [ 991.171854] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397790, 'name': ReconfigVM_Task, 'duration_secs': 0.258543} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 991.172332] env[62736]: DEBUG nova.network.neutron [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 991.172694] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 9aff2e37a3dc460db08998bcaac66e7b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 991.173509] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Reconfigured VM instance instance-00000062 to attach disk [datastore1] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 991.174061] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-2d83191a-3231-4f39-8429-8414fbc65448 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 991.182279] env[62736]: DEBUG nova.compute.manager [req-242faf3d-8286-475d-99e3-e76929a21ee3 req-2f3f599c-95b6-41c4-bf4f-cbe2e522a567 service nova] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Received event network-vif-deleted-923b9398-72d1-454e-bc28-1e0bdbf893e1 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 991.182279] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 991.182279] env[62736]: value = "task-397791" [ 991.182279] env[62736]: _type = "Task" [ 991.182279] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 991.183666] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9aff2e37a3dc460db08998bcaac66e7b [ 991.188904] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397791, 'name': Rename_Task} progress is 5%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 991.327486] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 991.327933] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 991.328140] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 991.328428] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-53f5a288-934b-4c85-a051-2353529b665f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 991.336917] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62d48b90-dab4-42dc-842a-b01cb25b874b {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 991.358536] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 8e17c8c1-e662-4695-9380-a43b7dd4ced6 could not be found. [ 991.358743] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 991.359075] env[62736]: INFO nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Took 0.03 seconds to destroy the instance on the hypervisor. [ 991.359326] env[62736]: DEBUG oslo.service.loopingcall [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 991.359639] env[62736]: DEBUG nova.compute.manager [-] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 991.359733] env[62736]: DEBUG nova.network.neutron [-] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 991.374504] env[62736]: DEBUG nova.network.neutron [-] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 991.374967] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 6d809489a35b4e7f8dcd7673e9030d17 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 991.381219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d809489a35b4e7f8dcd7673e9030d17 [ 991.677634] env[62736]: INFO nova.compute.manager [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 9f629513-ffbd-4879-949e-816c160149b2] Took 1.04 seconds to deallocate network for instance. [ 991.679329] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 6090c38ff15047cca3194c242cade58f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 991.689480] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397791, 'name': Rename_Task, 'duration_secs': 0.122344} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 991.689755] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 991.689991] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-3ecbbb2d-3e5e-42f4-b5b9-01db813a1953 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 991.697887] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 991.697887] env[62736]: value = "task-397792" [ 991.697887] env[62736]: _type = "Task" [ 991.697887] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 991.706422] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397792, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 991.713871] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6090c38ff15047cca3194c242cade58f [ 991.883217] env[62736]: DEBUG nova.network.neutron [-] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 991.883733] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg e6a8eda33df64ae8aab32c1de9471dc5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 991.892513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6a8eda33df64ae8aab32c1de9471dc5 [ 992.186958] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 19d39ce5ad2c4fe98515d121f2d19725 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 992.208166] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397792, 'name': PowerOnVM_Task} progress is 88%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 992.216235] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 19d39ce5ad2c4fe98515d121f2d19725 [ 992.390965] env[62736]: INFO nova.compute.manager [-] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Took 1.03 seconds to deallocate network for instance. [ 992.393364] env[62736]: DEBUG nova.compute.claims [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 992.393538] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 992.393759] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 992.395500] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg fd888b26296b438cbd19db22e0e2ce8e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 992.427165] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fd888b26296b438cbd19db22e0e2ce8e [ 992.707762] env[62736]: DEBUG oslo_vmware.api [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397792, 'name': PowerOnVM_Task, 'duration_secs': 0.610843} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 992.707762] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 992.708201] env[62736]: INFO nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Took 5.29 seconds to spawn the instance on the hypervisor. [ 992.708201] env[62736]: DEBUG nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 992.708963] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38520751-707a-43cd-96fc-30c58502a41c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.713444] env[62736]: INFO nova.scheduler.client.report [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance 9f629513-ffbd-4879-949e-816c160149b2 [ 992.719222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg a6b11fde57b64a0cb995c2e7ad4ff314 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 992.723527] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 7bba95e0088843d2b35c373517a9a460 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 992.734317] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6b11fde57b64a0cb995c2e7ad4ff314 [ 992.772239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bba95e0088843d2b35c373517a9a460 [ 992.934211] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df09a9cb-d69e-41dd-ba36-c5551dcc806d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.942128] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb259a23-981a-4782-b5f2-c5ac942d8cb3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.972117] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98a8c3d9-646a-41c1-93e0-147af2ebafe1 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.979768] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-913d16f2-1bcd-4dfd-b8c1-aab4210d6bbf {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 992.992880] env[62736]: DEBUG nova.compute.provider_tree [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 992.993438] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg e72462cb63684aca820b3dc302b04420 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 993.000445] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e72462cb63684aca820b3dc302b04420 [ 993.150003] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 62bc65702aa54d729f8aeea481429d5c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 993.158938] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 62bc65702aa54d729f8aeea481429d5c [ 993.220667] env[62736]: DEBUG oslo_concurrency.lockutils [None req-3256ee14-e0b8-452f-8842-69b3634f23b4 tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "9f629513-ffbd-4879-949e-816c160149b2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 15.250s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 993.231280] env[62736]: INFO nova.compute.manager [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Took 11.50 seconds to build instance. [ 993.231581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg b96da815c9aa4474894425f3916b55d5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 993.244691] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b96da815c9aa4474894425f3916b55d5 [ 993.496380] env[62736]: DEBUG nova.scheduler.client.report [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 993.498658] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 939ac48de3a5428aaee70d9249216535 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 993.511232] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 939ac48de3a5428aaee70d9249216535 [ 993.652270] env[62736]: INFO nova.compute.manager [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Rebuilding instance [ 993.688226] env[62736]: DEBUG nova.compute.manager [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 993.689084] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccdd6628-78ba-4e85-8811-9e6773a37ba5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 993.700728] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 0190d44377f944fa83d41065bf68bc05 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 993.734056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0190d44377f944fa83d41065bf68bc05 [ 993.734718] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cfd7e1b0-284d-47db-8d71-55a9f1e46bb7 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.006s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 994.001003] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.607s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 994.001641] env[62736]: ERROR nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Traceback (most recent call last): [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.driver.spawn(context, instance, image_meta, [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self._vmops.spawn(context, instance, image_meta, injected_files, [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] vm_ref = self.build_virtual_machine(instance, [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] vif_infos = vmwarevif.get_vif_info(self._session, [ 994.001641] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] for vif in network_info: [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return self._sync_wrapper(fn, *args, **kwargs) [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.wait() [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self[:] = self._gt.wait() [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return self._exit_event.wait() [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] current.throw(*self._exc) [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 994.002045] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] result = function(*args, **kwargs) [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] return func(*args, **kwargs) [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise e [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] nwinfo = self.network_api.allocate_for_instance( [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] created_port_ids = self._update_ports_for_instance( [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] with excutils.save_and_reraise_exception(): [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] self.force_reraise() [ 994.002457] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise self.value [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] updated_port = self._update_port( [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] _ensure_no_port_binding_failure(port) [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] raise exception.PortBindingFailed(port_id=port['id']) [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] nova.exception.PortBindingFailed: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. [ 994.002870] env[62736]: ERROR nova.compute.manager [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] [ 994.002870] env[62736]: DEBUG nova.compute.utils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 994.003827] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Build of instance 8e17c8c1-e662-4695-9380-a43b7dd4ced6 was re-scheduled: Binding failed for port 923b9398-72d1-454e-bc28-1e0bdbf893e1, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 994.004244] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 994.004465] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquiring lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 994.004608] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Acquired lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 994.004758] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 994.005141] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg f3a232968c6c49f2be742cc4f7798fd1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 994.013900] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3a232968c6c49f2be742cc4f7798fd1 [ 994.204756] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 994.205064] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-1827f8d9-a97b-47c2-a955-162b51262c99 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.213560] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 994.213560] env[62736]: value = "task-397793" [ 994.213560] env[62736]: _type = "Task" [ 994.213560] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 994.221693] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397793, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 994.473440] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "5732777e-b491-4e60-bc28-20f3f636f6fe" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 994.473664] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "5732777e-b491-4e60-bc28-20f3f636f6fe" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 994.474110] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b065771930f54728aae87133193e0abc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 994.483357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b065771930f54728aae87133193e0abc [ 994.521263] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 994.602116] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 994.602642] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 6452e49d76224ef885327f473b31ec0c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 994.611061] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6452e49d76224ef885327f473b31ec0c [ 994.723216] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397793, 'name': PowerOffVM_Task, 'duration_secs': 0.138836} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 994.723460] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 994.723681] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 994.724447] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc088c36-6c35-4e9e-b2f2-b0173909d9e3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.730804] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 994.731008] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-747776b1-2f13-4869-9bf9-4645c807aa17 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.755557] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 994.755913] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Deleting contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 994.755963] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Deleting the datastore file [datastore1] 267e30d2-c00c-4edf-8665-cfa88eba0938 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 994.756196] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-fbc153e6-b950-4ff0-a9e3-612089693e28 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 994.762177] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 994.762177] env[62736]: value = "task-397795" [ 994.762177] env[62736]: _type = "Task" [ 994.762177] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 994.769361] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397795, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 994.975996] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 994.977754] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ed74ecba49ea468f9f87812025825f57 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.009357] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed74ecba49ea468f9f87812025825f57 [ 995.104411] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Releasing lock "refresh_cache-8e17c8c1-e662-4695-9380-a43b7dd4ced6" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 995.104641] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 995.104825] env[62736]: DEBUG nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 995.104991] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 995.119334] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 995.119942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 7c2b777151ea417ebd203059bedef686 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.127018] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c2b777151ea417ebd203059bedef686 [ 995.271671] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397795, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086752} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 995.271921] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 995.272116] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Deleted contents of the VM from datastore datastore1 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 995.272322] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 995.273961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 9740ad3c8db74fbeb117c2eef6645f5f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.330180] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9740ad3c8db74fbeb117c2eef6645f5f [ 995.497076] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 995.497336] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 995.498763] env[62736]: INFO nova.compute.claims [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 995.500340] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 140f3c24e3174cae977915ae4c5faa50 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.531062] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 140f3c24e3174cae977915ae4c5faa50 [ 995.621815] env[62736]: DEBUG nova.network.neutron [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 995.622302] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg b994c2b3b965474ba5178f745633c429 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.629950] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b994c2b3b965474ba5178f745633c429 [ 995.778056] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 5a86f58e11e84484834c11363705e8b9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 995.805473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a86f58e11e84484834c11363705e8b9 [ 996.003354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg bcae9c0b170d4f9f9c6fa13b59059655 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 996.011341] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bcae9c0b170d4f9f9c6fa13b59059655 [ 996.125003] env[62736]: INFO nova.compute.manager [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] [instance: 8e17c8c1-e662-4695-9380-a43b7dd4ced6] Took 1.02 seconds to deallocate network for instance. [ 996.126635] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 405af88d93c94643bb0fb0458f35f91b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 996.159718] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 405af88d93c94643bb0fb0458f35f91b [ 996.304310] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 996.304773] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 996.305067] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 996.305378] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 996.305644] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 996.305982] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 996.306305] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 996.306582] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 996.306859] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 996.307143] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 996.307457] env[62736]: DEBUG nova.virt.hardware [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 996.308394] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11c877ef-3994-4fea-9224-0a3fd1dc6e65 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.316507] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d922441a-4670-4e48-a9af-2c10b52576ee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.331226] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance VIF info [] {{(pid=62736) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 996.338303] env[62736]: DEBUG oslo.service.loopingcall [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 996.338641] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Creating VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 996.338946] env[62736]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ececae31-2611-455d-8b98-7578349e9cae {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.356759] env[62736]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 996.356759] env[62736]: value = "task-397796" [ 996.356759] env[62736]: _type = "Task" [ 996.356759] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 996.365437] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397796, 'name': CreateVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 996.558844] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-580328bc-ee58-46a9-92f0-dcb2bb431f7d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.566057] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d75d65dd-125f-4253-ae14-4f9deb35f3e8 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.596084] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cef1b7b-6cc5-47ba-9b3f-ef8efe1fb39c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.602767] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03d07457-1344-4034-8c4e-14d11dbfbebe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.615297] env[62736]: DEBUG nova.compute.provider_tree [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 996.615775] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e9e37389edb247ffa242964edfc59c04 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 996.622671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e9e37389edb247ffa242964edfc59c04 [ 996.630731] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 90e8e67bd3af49b3b19bb915682ffe88 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 996.657954] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90e8e67bd3af49b3b19bb915682ffe88 [ 996.866579] env[62736]: DEBUG oslo_vmware.api [-] Task: {'id': task-397796, 'name': CreateVM_Task, 'duration_secs': 0.246237} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 996.866865] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Created VM on the ESX host {{(pid=62736) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 996.867149] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 996.867311] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 996.867675] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 996.867925] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-237c9131-c2c5-4fb3-bf06-8526d991b708 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 996.872043] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 996.872043] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]522be06d-c45b-eecf-d657-cf7475f31929" [ 996.872043] env[62736]: _type = "Task" [ 996.872043] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 996.879080] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]522be06d-c45b-eecf-d657-cf7475f31929, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 997.119057] env[62736]: DEBUG nova.scheduler.client.report [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 997.121447] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ed62efb9c33b45e79aa1500bcfd7cec8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 997.132684] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ed62efb9c33b45e79aa1500bcfd7cec8 [ 997.151764] env[62736]: INFO nova.scheduler.client.report [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Deleted allocations for instance 8e17c8c1-e662-4695-9380-a43b7dd4ced6 [ 997.157497] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Expecting reply to msg 72bc85da40b246ddbc49970a2bdeb1d0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 997.171109] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 72bc85da40b246ddbc49970a2bdeb1d0 [ 997.382894] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]522be06d-c45b-eecf-d657-cf7475f31929, 'name': SearchDatastore_Task, 'duration_secs': 0.010763} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 997.383304] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 997.383629] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Processing image 81867c62-ef8e-483f-bfd2-854abdcd6db5 {{(pid=62736) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 997.383969] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 997.384230] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 997.384485] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 997.384841] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f7a28d67-30d7-484d-a8fc-5f05b9551a77 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.392746] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=62736) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 997.393016] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=62736) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 997.393766] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d958d3ac-078b-4c06-9f87-f27d173fab66 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.399307] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 997.399307] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52ff29d3-e0f4-c1ec-2e42-455bf2b9ec92" [ 997.399307] env[62736]: _type = "Task" [ 997.399307] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 997.406682] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52ff29d3-e0f4-c1ec-2e42-455bf2b9ec92, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 997.624434] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.127s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.625046] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 997.626836] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 06e390f40239413790d6636e0d372db4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 997.656230] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 06e390f40239413790d6636e0d372db4 [ 997.659406] env[62736]: DEBUG oslo_concurrency.lockutils [None req-73f389c2-ca88-49a0-a802-59cd15dd39ce tempest-ServerDiskConfigTestJSON-215884773 tempest-ServerDiskConfigTestJSON-215884773-project-member] Lock "8e17c8c1-e662-4695-9380-a43b7dd4ced6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 16.901s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 997.910293] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52ff29d3-e0f4-c1ec-2e42-455bf2b9ec92, 'name': SearchDatastore_Task, 'duration_secs': 0.00832} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 997.911200] env[62736]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-88e27fbb-b70a-4d25-a24a-3fff64d90d40 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 997.916898] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 997.916898] env[62736]: value = "session[520d8be4-1196-41a6-be08-036002f9b00f]52830e0f-c6ae-4556-2277-c3ab69ceb6c2" [ 997.916898] env[62736]: _type = "Task" [ 997.916898] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 997.930672] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52830e0f-c6ae-4556-2277-c3ab69ceb6c2, 'name': SearchDatastore_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 998.130550] env[62736]: DEBUG nova.compute.utils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 998.131242] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 82c6fbb775da4d96997e0c63616c07bc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 998.132233] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 998.132404] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 998.142376] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 82c6fbb775da4d96997e0c63616c07bc [ 998.197303] env[62736]: DEBUG nova.policy [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '949bec663bc44ef3b1d3eb8c8f6912a9', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'adf774b7c36f46219e7dfdec663fe624', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 998.429113] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': session[520d8be4-1196-41a6-be08-036002f9b00f]52830e0f-c6ae-4556-2277-c3ab69ceb6c2, 'name': SearchDatastore_Task, 'duration_secs': 0.008581} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 998.429390] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Releasing lock "[datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 998.429686] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Copying Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 998.429943] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-af714bb5-f4b8-4a57-823c-0a33b4a5c651 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.439429] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 998.439429] env[62736]: value = "task-397797" [ 998.439429] env[62736]: _type = "Task" [ 998.439429] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 998.448116] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397797, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 998.465408] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Successfully created port: b10809fc-4402-4884-a6c6-a6a4a7484394 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 998.635957] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 998.637792] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 5668a5ba4c60402bbe1aabca2b1a8cdc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 998.679974] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5668a5ba4c60402bbe1aabca2b1a8cdc [ 998.948544] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397797, 'name': CopyVirtualDisk_Task, 'duration_secs': 0.458551} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 998.948892] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Copied Virtual Disk [datastore2] devstack-image-cache_base/81867c62-ef8e-483f-bfd2-854abdcd6db5/81867c62-ef8e-483f-bfd2-854abdcd6db5.vmdk to [datastore2] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk {{(pid=62736) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1424}} [ 998.949113] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Extending root virtual disk to 1048576 {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:167}} [ 998.949354] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.ExtendVirtualDisk_Task with opID=oslo.vmware-4db2ae73-fb6f-413d-9ae0-bba8acf5b535 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 998.956055] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 998.956055] env[62736]: value = "task-397798" [ 998.956055] env[62736]: _type = "Task" [ 998.956055] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 998.964696] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397798, 'name': ExtendVirtualDisk_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 999.142940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b29a74e8cb3d45be829ab90f804d71a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 999.187960] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b29a74e8cb3d45be829ab90f804d71a3 [ 999.339921] env[62736]: DEBUG nova.compute.manager [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Received event network-changed-b10809fc-4402-4884-a6c6-a6a4a7484394 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 999.340136] env[62736]: DEBUG nova.compute.manager [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Refreshing instance network info cache due to event network-changed-b10809fc-4402-4884-a6c6-a6a4a7484394. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 999.340332] env[62736]: DEBUG oslo_concurrency.lockutils [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] Acquiring lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.340481] env[62736]: DEBUG oslo_concurrency.lockutils [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] Acquired lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 999.340660] env[62736]: DEBUG nova.network.neutron [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Refreshing network info cache for port b10809fc-4402-4884-a6c6-a6a4a7484394 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 999.341073] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] Expecting reply to msg adb9e94fbdd943e7b94c69569a03b02b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 999.347751] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg adb9e94fbdd943e7b94c69569a03b02b [ 999.465713] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397798, 'name': ExtendVirtualDisk_Task, 'duration_secs': 0.068711} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 999.465979] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Extended root virtual disk {{(pid=62736) _extend_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vmops.py:189}} [ 999.466746] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e782e905-7e27-4c0b-b655-1196bc80852d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.487645] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Reconfiguring VM instance instance-00000062 to attach disk [datastore2] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 999.488256] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-a3b23943-95f6-4a8c-9818-0a46a5a80ef7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.504180] env[62736]: ERROR nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 999.504180] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.504180] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.504180] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.504180] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.504180] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.504180] env[62736]: ERROR nova.compute.manager raise self.value [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.504180] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 999.504180] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.504180] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 999.504667] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.504667] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 999.504667] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 999.504667] env[62736]: ERROR nova.compute.manager [ 999.504667] env[62736]: Traceback (most recent call last): [ 999.504667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 999.504667] env[62736]: listener.cb(fileno) [ 999.504667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.504667] env[62736]: result = function(*args, **kwargs) [ 999.504667] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 999.504667] env[62736]: return func(*args, **kwargs) [ 999.504667] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.504667] env[62736]: raise e [ 999.504667] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.504667] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 999.504667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.504667] env[62736]: created_port_ids = self._update_ports_for_instance( [ 999.504667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.504667] env[62736]: with excutils.save_and_reraise_exception(): [ 999.504667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.504667] env[62736]: self.force_reraise() [ 999.504667] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.504667] env[62736]: raise self.value [ 999.504667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.504667] env[62736]: updated_port = self._update_port( [ 999.504667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.504667] env[62736]: _ensure_no_port_binding_failure(port) [ 999.504667] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.504667] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 999.505553] env[62736]: nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 999.505553] env[62736]: Removing descriptor: 19 [ 999.509634] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 999.509634] env[62736]: value = "task-397799" [ 999.509634] env[62736]: _type = "Task" [ 999.509634] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 999.517521] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397799, 'name': ReconfigVM_Task} progress is 6%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 999.646203] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 999.672374] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 999.672667] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 999.672864] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 999.673062] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 999.673208] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 999.673365] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 999.673582] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 999.673750] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 999.673914] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 999.674086] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 999.674256] env[62736]: DEBUG nova.virt.hardware [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 999.675125] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4d3dabb-ebd0-43a3-88fa-4ab6d05c7037 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.684687] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51b1fa23-275f-48c2-afa1-ce68f721a6fe {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 999.699212] env[62736]: ERROR nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Traceback (most recent call last): [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] yield resources [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.driver.spawn(context, instance, image_meta, [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] vm_ref = self.build_virtual_machine(instance, [ 999.699212] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] vif_infos = vmwarevif.get_vif_info(self._session, [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] for vif in network_info: [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return self._sync_wrapper(fn, *args, **kwargs) [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.wait() [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self[:] = self._gt.wait() [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return self._exit_event.wait() [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 999.699648] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] current.throw(*self._exc) [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] result = function(*args, **kwargs) [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return func(*args, **kwargs) [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise e [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] nwinfo = self.network_api.allocate_for_instance( [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] created_port_ids = self._update_ports_for_instance( [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] with excutils.save_and_reraise_exception(): [ 999.700094] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.force_reraise() [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise self.value [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] updated_port = self._update_port( [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] _ensure_no_port_binding_failure(port) [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise exception.PortBindingFailed(port_id=port['id']) [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 999.700718] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] [ 999.700718] env[62736]: INFO nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Terminating instance [ 999.701901] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 999.857978] env[62736]: DEBUG nova.network.neutron [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 999.938608] env[62736]: DEBUG nova.network.neutron [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 999.939160] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] Expecting reply to msg 3d5a31043697482780670c1e977fda22 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 999.948541] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3d5a31043697482780670c1e977fda22 [ 1000.019891] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397799, 'name': ReconfigVM_Task, 'duration_secs': 0.292021} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1000.020259] env[62736]: DEBUG nova.virt.vmwareapi.volumeops [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Reconfigured VM instance instance-00000062 to attach disk [datastore2] 267e30d2-c00c-4edf-8665-cfa88eba0938/267e30d2-c00c-4edf-8665-cfa88eba0938.vmdk or device None with type sparse {{(pid=62736) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1000.020891] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-0105ae73-1167-4718-8ac7-7ea767449801 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.027629] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 1000.027629] env[62736]: value = "task-397800" [ 1000.027629] env[62736]: _type = "Task" [ 1000.027629] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1000.035607] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397800, 'name': Rename_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1000.441719] env[62736]: DEBUG oslo_concurrency.lockutils [req-86e13251-33ee-411f-b541-57d91dae90b4 req-a8c4c66d-2828-4e52-8a8d-d947cdf62a4c service nova] Releasing lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1000.442174] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1000.442363] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1000.442801] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg c5c291e8ffac4e3cab96bb02b4b7fecf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1000.449998] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c5c291e8ffac4e3cab96bb02b4b7fecf [ 1000.537244] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397800, 'name': Rename_Task, 'duration_secs': 0.134587} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1000.537616] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powering on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 1000.537912] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-3948f2f9-f37d-4778-9bf0-b2ec410b910d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1000.544082] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 1000.544082] env[62736]: value = "task-397801" [ 1000.544082] env[62736]: _type = "Task" [ 1000.544082] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1000.551516] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397801, 'name': PowerOnVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1000.961405] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1001.037179] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1001.037791] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 783345e0cdde48aca3d38c31386c5a76 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1001.049256] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 783345e0cdde48aca3d38c31386c5a76 [ 1001.054147] env[62736]: DEBUG oslo_vmware.api [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397801, 'name': PowerOnVM_Task, 'duration_secs': 0.431155} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1001.054436] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powered on the VM {{(pid=62736) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 1001.054669] env[62736]: DEBUG nova.compute.manager [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Checking state {{(pid=62736) _get_power_state /opt/stack/nova/nova/compute/manager.py:1791}} [ 1001.055494] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee9ca937-91ed-4d5d-bff4-10eb4d1ecbb5 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.063321] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 6d47ffae77974f30a1c64386fabc8404 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1001.096436] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6d47ffae77974f30a1c64386fabc8404 [ 1001.365624] env[62736]: DEBUG nova.compute.manager [req-b202f64d-46b6-4fcb-938b-0844dec9a9b1 req-6e81b4bf-a79a-41ee-9211-ba913004fefa service nova] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Received event network-vif-deleted-b10809fc-4402-4884-a6c6-a6a4a7484394 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1001.540876] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1001.541420] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1001.541685] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1001.542065] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ae474c63-460e-41d3-89d5-7a8e2972434d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.550990] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c200c2d-1854-496b-9d82-52eed5dafac9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1001.575368] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5732777e-b491-4e60-bc28-20f3f636f6fe could not be found. [ 1001.575606] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1001.575786] env[62736]: INFO nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1001.576070] env[62736]: DEBUG oslo.service.loopingcall [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1001.577877] env[62736]: DEBUG nova.compute.manager [-] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1001.577976] env[62736]: DEBUG nova.network.neutron [-] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1001.579721] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1001.579935] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1001.580115] env[62736]: DEBUG nova.objects.instance [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Trying to apply a migration context that does not seem to be set for this instance {{(pid=62736) apply_migration_context /opt/stack/nova/nova/objects/instance.py:1067}} [ 1001.581730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg ccc78dbb43774eb7a608160b346706ef in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1001.596682] env[62736]: DEBUG nova.network.neutron [-] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1001.597205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg d0ad7271875b415d8dcc1fe392ad2ef8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1001.605604] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0ad7271875b415d8dcc1fe392ad2ef8 [ 1001.613798] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccc78dbb43774eb7a608160b346706ef [ 1001.772620] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 66d42abd16094de8b58b4a4726b96d93 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1001.783016] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66d42abd16094de8b58b4a4726b96d93 [ 1002.086222] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 99726932133549e0b85e8cafeb7e231c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.093849] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99726932133549e0b85e8cafeb7e231c [ 1002.098921] env[62736]: DEBUG nova.network.neutron [-] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1002.099279] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg ca62d601ebdf4ad3b13b797513494b83 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.106511] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ca62d601ebdf4ad3b13b797513494b83 [ 1002.275002] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "267e30d2-c00c-4edf-8665-cfa88eba0938" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1002.275268] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1002.275473] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "267e30d2-c00c-4edf-8665-cfa88eba0938-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1002.275655] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1002.275822] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1002.278021] env[62736]: INFO nova.compute.manager [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Terminating instance [ 1002.279627] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "refresh_cache-267e30d2-c00c-4edf-8665-cfa88eba0938" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1002.279776] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquired lock "refresh_cache-267e30d2-c00c-4edf-8665-cfa88eba0938" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1002.279937] env[62736]: DEBUG nova.network.neutron [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1002.280349] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 3987ddf830a6492ba4e8ce13c59a49d2 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.286518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3987ddf830a6492ba4e8ce13c59a49d2 [ 1002.589964] env[62736]: DEBUG oslo_concurrency.lockutils [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.finish_evacuation" :: held 1.010s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1002.590348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-2fa980b1-1f3e-4df1-8376-2a7a4c5f21da tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg fb7b900f96954851aa01f7701e36afc1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.600205] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb7b900f96954851aa01f7701e36afc1 [ 1002.601188] env[62736]: INFO nova.compute.manager [-] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Took 1.02 seconds to deallocate network for instance. [ 1002.603774] env[62736]: DEBUG nova.compute.claims [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1002.603943] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1002.604164] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1002.605824] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg e2cdc5e18ced415aa2a6c46ac111a470 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.634986] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e2cdc5e18ced415aa2a6c46ac111a470 [ 1002.797349] env[62736]: DEBUG nova.network.neutron [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1002.848452] env[62736]: DEBUG nova.network.neutron [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1002.848973] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 737ce298c53349cca7f08d6e7b0c4cac in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1002.857818] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 737ce298c53349cca7f08d6e7b0c4cac [ 1003.150966] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72ddb26f-8c1a-4b3c-84c3-16594687d349 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.158683] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-511cdaaf-3e67-448a-b93d-550b10e74694 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.189194] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc877cf1-ffee-439b-99b6-d99723918036 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.196191] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f65fd023-f2d6-4bcc-a8ed-3b4546bc031d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.209059] env[62736]: DEBUG nova.compute.provider_tree [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1003.209591] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 3c254cfae37d4874b0106599e695b8ab in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1003.216961] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3c254cfae37d4874b0106599e695b8ab [ 1003.351731] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Releasing lock "refresh_cache-267e30d2-c00c-4edf-8665-cfa88eba0938" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1003.352179] env[62736]: DEBUG nova.compute.manager [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1003.352372] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1003.353240] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db75d421-dcb2-469b-ace5-8b47d53cc334 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.360987] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powering off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 1003.361218] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-bff9adef-c118-425b-8f58-270b1efe1246 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.366580] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 1003.366580] env[62736]: value = "task-397802" [ 1003.366580] env[62736]: _type = "Task" [ 1003.366580] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1003.374165] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397802, 'name': PowerOffVM_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1003.712199] env[62736]: DEBUG nova.scheduler.client.report [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1003.714575] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 9538d0c726c3455fb1e0a66425bb0224 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1003.725922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9538d0c726c3455fb1e0a66425bb0224 [ 1003.877109] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397802, 'name': PowerOffVM_Task, 'duration_secs': 0.186553} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1003.877405] env[62736]: DEBUG nova.virt.vmwareapi.vm_util [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Powered off the VM {{(pid=62736) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 1003.877574] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Unregistering the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1003.877828] env[62736]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-cf56b53f-9ba2-48af-8a50-e39e20eff049 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.903214] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Unregistered the VM {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1003.903438] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Deleting contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1003.903618] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Deleting the datastore file [datastore2] 267e30d2-c00c-4edf-8665-cfa88eba0938 {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1003.903888] env[62736]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-29ee3f35-2800-40f9-a372-806de4c5974e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1003.911071] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for the task: (returnval){ [ 1003.911071] env[62736]: value = "task-397804" [ 1003.911071] env[62736]: _type = "Task" [ 1003.911071] env[62736]: } to complete. {{(pid=62736) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1003.919332] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397804, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1004.217209] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.613s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1004.236205] env[62736]: ERROR nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Traceback (most recent call last): [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.driver.spawn(context, instance, image_meta, [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] vm_ref = self.build_virtual_machine(instance, [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] vif_infos = vmwarevif.get_vif_info(self._session, [ 1004.236205] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] for vif in network_info: [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return self._sync_wrapper(fn, *args, **kwargs) [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.wait() [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self[:] = self._gt.wait() [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return self._exit_event.wait() [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] current.throw(*self._exc) [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1004.236532] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] result = function(*args, **kwargs) [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] return func(*args, **kwargs) [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise e [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] nwinfo = self.network_api.allocate_for_instance( [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] created_port_ids = self._update_ports_for_instance( [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] with excutils.save_and_reraise_exception(): [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] self.force_reraise() [ 1004.236952] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise self.value [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] updated_port = self._update_port( [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] _ensure_no_port_binding_failure(port) [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] raise exception.PortBindingFailed(port_id=port['id']) [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] nova.exception.PortBindingFailed: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. [ 1004.237311] env[62736]: ERROR nova.compute.manager [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] [ 1004.237311] env[62736]: DEBUG nova.compute.utils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1004.237630] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Build of instance 5732777e-b491-4e60-bc28-20f3f636f6fe was re-scheduled: Binding failed for port b10809fc-4402-4884-a6c6-a6a4a7484394, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1004.237630] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1004.237630] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquiring lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1004.237630] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Acquired lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1004.237865] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1004.237865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg ced305cc86e54570b0cbc801c410f5fc in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1004.237865] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ced305cc86e54570b0cbc801c410f5fc [ 1004.420896] env[62736]: DEBUG oslo_vmware.api [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Task: {'id': task-397804, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.105432} completed successfully. {{(pid=62736) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1004.421154] env[62736]: DEBUG nova.virt.vmwareapi.ds_util [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Deleted the datastore file {{(pid=62736) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1004.421269] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Deleted contents of the VM from datastore datastore2 {{(pid=62736) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1004.421438] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1004.421605] env[62736]: INFO nova.compute.manager [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Took 1.07 seconds to destroy the instance on the hypervisor. [ 1004.422154] env[62736]: DEBUG oslo.service.loopingcall [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1004.422154] env[62736]: DEBUG nova.compute.manager [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1004.422154] env[62736]: DEBUG nova.network.neutron [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1004.436117] env[62736]: DEBUG nova.network.neutron [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1004.436581] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 0c49fad01ab64e6da58e72c82125034a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1004.442942] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0c49fad01ab64e6da58e72c82125034a [ 1004.737457] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1004.806135] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1004.806632] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 26b17dc2847c4b02858d0f67e5836f10 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1004.814641] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26b17dc2847c4b02858d0f67e5836f10 [ 1004.938565] env[62736]: DEBUG nova.network.neutron [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1004.938995] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 794ca974088441c3ac9fb0e435649bae in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1004.947132] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 794ca974088441c3ac9fb0e435649bae [ 1005.309578] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Releasing lock "refresh_cache-5732777e-b491-4e60-bc28-20f3f636f6fe" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1005.310011] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1005.310011] env[62736]: DEBUG nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1005.310166] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1005.324816] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1005.325324] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 5d18ef10e8814b82b19a702740f717eb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1005.331536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d18ef10e8814b82b19a702740f717eb [ 1005.441221] env[62736]: INFO nova.compute.manager [-] [instance: 267e30d2-c00c-4edf-8665-cfa88eba0938] Took 1.02 seconds to deallocate network for instance. [ 1005.445123] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 6ecda36f45874b62a445cbb81edbd7c1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1005.471414] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6ecda36f45874b62a445cbb81edbd7c1 [ 1005.827859] env[62736]: DEBUG nova.network.neutron [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1005.828409] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 5d2a3ccb084e4f96a7471123fbb62541 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1005.836176] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5d2a3ccb084e4f96a7471123fbb62541 [ 1005.947386] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1005.947671] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1005.947895] env[62736]: DEBUG nova.objects.instance [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lazy-loading 'resources' on Instance uuid 267e30d2-c00c-4edf-8665-cfa88eba0938 {{(pid=62736) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 1005.948261] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg e5f8ad2f2a29474e9fc0db7874d5eb99 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1005.956795] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e5f8ad2f2a29474e9fc0db7874d5eb99 [ 1006.330806] env[62736]: INFO nova.compute.manager [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] [instance: 5732777e-b491-4e60-bc28-20f3f636f6fe] Took 1.02 seconds to deallocate network for instance. [ 1006.332574] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b138dc241d92471d9d233001af951cdd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1006.363669] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b138dc241d92471d9d233001af951cdd [ 1006.484489] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f489aa14-b1d6-4727-9ad3-bbd7490032bd {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.491805] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ee9fe65-5dee-4f53-aa5a-dfa3575a3009 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.520748] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20a3d611-5c57-4a62-9c80-2ed51b063d1c {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.527167] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23793f84-c686-4138-9bc1-90408652ede9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1006.539507] env[62736]: DEBUG nova.compute.provider_tree [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1006.539967] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg da0030e53f754960960797d57b162267 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1006.546552] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da0030e53f754960960797d57b162267 [ 1006.837044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg 7153583c558149d59799f693b1f3232c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1006.866282] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7153583c558149d59799f693b1f3232c [ 1007.043519] env[62736]: DEBUG nova.scheduler.client.report [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1007.045730] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 3de45fd3814e49c3bce9a85ca5cdf59d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1007.056128] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3de45fd3814e49c3bce9a85ca5cdf59d [ 1007.357456] env[62736]: INFO nova.scheduler.client.report [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Deleted allocations for instance 5732777e-b491-4e60-bc28-20f3f636f6fe [ 1007.363366] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Expecting reply to msg b5ef879306d04e2c975180e979657602 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1007.372848] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b5ef879306d04e2c975180e979657602 [ 1007.547811] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.600s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1007.566144] env[62736]: INFO nova.scheduler.client.report [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Deleted allocations for instance 267e30d2-c00c-4edf-8665-cfa88eba0938 [ 1007.569138] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 899f7d2459a84367a5f3a114b61d354d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1007.606058] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 899f7d2459a84367a5f3a114b61d354d [ 1007.866514] env[62736]: DEBUG oslo_concurrency.lockutils [None req-cf938004-bd7c-419e-8bd3-232528ea955c tempest-DeleteServersTestJSON-57768650 tempest-DeleteServersTestJSON-57768650-project-member] Lock "5732777e-b491-4e60-bc28-20f3f636f6fe" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.393s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1008.073178] env[62736]: DEBUG oslo_concurrency.lockutils [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Lock "267e30d2-c00c-4edf-8665-cfa88eba0938" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 5.798s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1008.073519] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-6588929f-e07f-4866-9ed1-789b5baa26b0 tempest-ServerShowV254Test-392752660 tempest-ServerShowV254Test-392752660-project-member] Expecting reply to msg 20a8e19717234ec3969974215512070e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1008.082943] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20a8e19717234ec3969974215512070e [ 1012.356923] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "14bc65e1-9305-4d75-8d27-d3077e7120c9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1012.357253] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "14bc65e1-9305-4d75-8d27-d3077e7120c9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1012.357623] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 351cb3223ddb4ab4bd5a3f3ebcbf5213 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1012.370873] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 351cb3223ddb4ab4bd5a3f3ebcbf5213 [ 1012.859866] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1012.861866] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg cfb62a5e5f594fe290a1d644004277bf in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1012.895714] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cfb62a5e5f594fe290a1d644004277bf [ 1013.381074] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.381368] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.382914] env[62736]: INFO nova.compute.claims [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1013.384518] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 1233017dd85b433693f0eefa070005f4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1013.415466] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1233017dd85b433693f0eefa070005f4 [ 1013.888151] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 2d77b0aa5105455cae079af334c52532 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1013.896956] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d77b0aa5105455cae079af334c52532 [ 1014.419599] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-52dd3eee-67ff-488d-a7c2-456c1b9e6496 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.427280] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45c0398b-2abd-4d73-a316-a3e65fe48cdc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.457004] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6077786b-58d6-4f85-a451-aca89e28c38e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.463597] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bddcc99d-af76-4efe-88a3-e930f35ae355 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1014.476019] env[62736]: DEBUG nova.compute.provider_tree [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1014.476475] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg f8bcd073b674420f95470586ac0da6be in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1014.483319] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8bcd073b674420f95470586ac0da6be [ 1014.978711] env[62736]: DEBUG nova.scheduler.client.report [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1014.981280] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 526c1f3226b74faeb52197e7d18eb59b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1014.992629] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 526c1f3226b74faeb52197e7d18eb59b [ 1015.483986] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.102s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1015.484562] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1015.486333] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 00069ee75c214870afb5e0cb723c948c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1015.516979] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 00069ee75c214870afb5e0cb723c948c [ 1015.989689] env[62736]: DEBUG nova.compute.utils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1015.990332] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 0903c3abf1f24af3a5199efdd1365671 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1015.991280] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1015.991453] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1016.003098] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0903c3abf1f24af3a5199efdd1365671 [ 1016.038064] env[62736]: DEBUG nova.policy [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88899e5c37af4416a1d5e65b61ca6218', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'af0fcc89cc8548f7b5f9689df14993c4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 1016.303203] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Successfully created port: 67538936-c0fe-4f58-b367-f771cadee631 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1016.494883] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1016.496734] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 5a04c764844b4cc399123c90e230d5bd in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1016.528284] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a04c764844b4cc399123c90e230d5bd [ 1016.906570] env[62736]: DEBUG nova.compute.manager [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Received event network-changed-67538936-c0fe-4f58-b367-f771cadee631 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1016.906777] env[62736]: DEBUG nova.compute.manager [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Refreshing instance network info cache due to event network-changed-67538936-c0fe-4f58-b367-f771cadee631. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1016.906991] env[62736]: DEBUG oslo_concurrency.lockutils [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] Acquiring lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1016.907129] env[62736]: DEBUG oslo_concurrency.lockutils [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] Acquired lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1016.907286] env[62736]: DEBUG nova.network.neutron [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Refreshing network info cache for port 67538936-c0fe-4f58-b367-f771cadee631 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1016.907755] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] Expecting reply to msg f955b46143b8420fb30d953aa2d721c7 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1016.914716] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f955b46143b8420fb30d953aa2d721c7 [ 1017.001827] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg ab13cbad4e7644399597e0fb8675768d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1017.031354] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab13cbad4e7644399597e0fb8675768d [ 1017.064411] env[62736]: ERROR nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1017.064411] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1017.064411] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1017.064411] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1017.064411] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1017.064411] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1017.064411] env[62736]: ERROR nova.compute.manager raise self.value [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1017.064411] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 1017.064411] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1017.064411] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1017.064949] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1017.064949] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1017.064949] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1017.064949] env[62736]: ERROR nova.compute.manager [ 1017.064949] env[62736]: Traceback (most recent call last): [ 1017.064949] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1017.064949] env[62736]: listener.cb(fileno) [ 1017.064949] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1017.064949] env[62736]: result = function(*args, **kwargs) [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1017.064949] env[62736]: return func(*args, **kwargs) [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1017.064949] env[62736]: raise e [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1017.064949] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1017.064949] env[62736]: created_port_ids = self._update_ports_for_instance( [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1017.064949] env[62736]: with excutils.save_and_reraise_exception(): [ 1017.064949] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1017.064949] env[62736]: self.force_reraise() [ 1017.064949] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1017.064949] env[62736]: raise self.value [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1017.064949] env[62736]: updated_port = self._update_port( [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1017.064949] env[62736]: _ensure_no_port_binding_failure(port) [ 1017.064949] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1017.064949] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 1017.065769] env[62736]: nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1017.065769] env[62736]: Removing descriptor: 19 [ 1017.424636] env[62736]: DEBUG nova.network.neutron [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1017.492046] env[62736]: DEBUG nova.network.neutron [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1017.492608] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] Expecting reply to msg 49f511857d154e8cb84b13c5bacef46f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1017.500038] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 49f511857d154e8cb84b13c5bacef46f [ 1017.504664] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1017.532034] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1017.532034] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1017.532034] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1017.532263] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1017.532263] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1017.532263] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1017.532263] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1017.532263] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1017.532443] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1017.532443] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1017.532443] env[62736]: DEBUG nova.virt.hardware [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1017.532443] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e2f9053-6fc7-49a1-85f2-91f68dbbc130 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.539318] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-704d0de4-daa8-49cf-bc01-d4c7ad366cfc {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1017.554117] env[62736]: ERROR nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Traceback (most recent call last): [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] yield resources [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.driver.spawn(context, instance, image_meta, [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] vm_ref = self.build_virtual_machine(instance, [ 1017.554117] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] vif_infos = vmwarevif.get_vif_info(self._session, [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] for vif in network_info: [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return self._sync_wrapper(fn, *args, **kwargs) [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.wait() [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self[:] = self._gt.wait() [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return self._exit_event.wait() [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1017.554550] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] current.throw(*self._exc) [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] result = function(*args, **kwargs) [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return func(*args, **kwargs) [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise e [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] nwinfo = self.network_api.allocate_for_instance( [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] created_port_ids = self._update_ports_for_instance( [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] with excutils.save_and_reraise_exception(): [ 1017.554949] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.force_reraise() [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise self.value [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] updated_port = self._update_port( [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] _ensure_no_port_binding_failure(port) [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise exception.PortBindingFailed(port_id=port['id']) [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1017.555332] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] [ 1017.555332] env[62736]: INFO nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Terminating instance [ 1017.556421] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1017.994800] env[62736]: DEBUG oslo_concurrency.lockutils [req-5e805cc0-f483-41b0-937f-d03b29d15cd4 req-1c080dd2-e711-45b5-9ad0-c62cd56e8873 service nova] Releasing lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1017.995202] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquired lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1017.995387] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1017.995830] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 0407ac85f0d74b3ba995b7c59d17987b in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1018.002940] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0407ac85f0d74b3ba995b7c59d17987b [ 1018.512628] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1018.583844] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1018.584380] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg bfea0ec5c36c43d4a9d9a275821e1f6a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1018.592407] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bfea0ec5c36c43d4a9d9a275821e1f6a [ 1018.933936] env[62736]: DEBUG nova.compute.manager [req-abf504e9-72b0-4a5e-91db-c0ba54e7a33a req-17c271d6-3154-43d8-bff2-2ba2269e0f24 service nova] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Received event network-vif-deleted-67538936-c0fe-4f58-b367-f771cadee631 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1019.086345] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Releasing lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1019.086784] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1019.086977] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1019.087286] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0b975d1d-efcb-49e6-a174-02e10e7c8940 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.096689] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b797ffeb-977b-4e05-9aac-86fc58f47dee {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1019.117920] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 14bc65e1-9305-4d75-8d27-d3077e7120c9 could not be found. [ 1019.118211] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1019.118399] env[62736]: INFO nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1019.118636] env[62736]: DEBUG oslo.service.loopingcall [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1019.118856] env[62736]: DEBUG nova.compute.manager [-] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1019.118952] env[62736]: DEBUG nova.network.neutron [-] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1019.133946] env[62736]: DEBUG nova.network.neutron [-] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1019.134444] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 777d232d772a441f98e0c8f2dd8f6b2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1019.142044] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 777d232d772a441f98e0c8f2dd8f6b2d [ 1019.636797] env[62736]: DEBUG nova.network.neutron [-] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1019.637274] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 4fde053a5b314718a16fce97f32083c8 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1019.645778] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4fde053a5b314718a16fce97f32083c8 [ 1020.139370] env[62736]: INFO nova.compute.manager [-] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Took 1.02 seconds to deallocate network for instance. [ 1020.141743] env[62736]: DEBUG nova.compute.claims [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1020.141933] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.142148] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1020.143993] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg fdaa5f730fb84c6abdf68767e8d27f2d in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1020.179892] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fdaa5f730fb84c6abdf68767e8d27f2d [ 1020.677376] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1155c76e-4627-4704-96f9-aad260fcdd76 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.684812] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3621e771-41cc-4747-9198-3a13dfcacf53 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.714987] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33308140-3083-43b3-8791-752c29abeb7a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.721839] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-439adc42-c7a3-4e7d-b273-7b975ed4f1f7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.734346] env[62736]: DEBUG nova.compute.provider_tree [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1020.734823] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 3df9600876b14749865550438b20fb65 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1020.741913] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3df9600876b14749865550438b20fb65 [ 1021.237501] env[62736]: DEBUG nova.scheduler.client.report [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1021.240219] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg c0dd1e187d114a7c8e87961cc8fc303a in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1021.254017] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0dd1e187d114a7c8e87961cc8fc303a [ 1021.743299] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.601s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1021.743965] env[62736]: ERROR nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Traceback (most recent call last): [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.driver.spawn(context, instance, image_meta, [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] vm_ref = self.build_virtual_machine(instance, [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] vif_infos = vmwarevif.get_vif_info(self._session, [ 1021.743965] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] for vif in network_info: [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return self._sync_wrapper(fn, *args, **kwargs) [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.wait() [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self[:] = self._gt.wait() [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return self._exit_event.wait() [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] current.throw(*self._exc) [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1021.744372] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] result = function(*args, **kwargs) [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] return func(*args, **kwargs) [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise e [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] nwinfo = self.network_api.allocate_for_instance( [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] created_port_ids = self._update_ports_for_instance( [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] with excutils.save_and_reraise_exception(): [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] self.force_reraise() [ 1021.744745] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise self.value [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] updated_port = self._update_port( [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] _ensure_no_port_binding_failure(port) [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] raise exception.PortBindingFailed(port_id=port['id']) [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] nova.exception.PortBindingFailed: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. [ 1021.745102] env[62736]: ERROR nova.compute.manager [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] [ 1021.745102] env[62736]: DEBUG nova.compute.utils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1021.746367] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Build of instance 14bc65e1-9305-4d75-8d27-d3077e7120c9 was re-scheduled: Binding failed for port 67538936-c0fe-4f58-b367-f771cadee631, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1021.746766] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1021.746994] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1021.747139] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquired lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1021.747295] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1021.747695] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 5768e6721dc64d3ba55a21279f5ac582 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1021.754272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5768e6721dc64d3ba55a21279f5ac582 [ 1022.264372] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1022.345697] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1022.346229] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg fb65a45ede9142ef8c888487f88d3d9f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1022.354239] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb65a45ede9142ef8c888487f88d3d9f [ 1022.848557] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Releasing lock "refresh_cache-14bc65e1-9305-4d75-8d27-d3077e7120c9" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1022.848929] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1022.849001] env[62736]: DEBUG nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1022.849135] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1022.863218] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1022.863732] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 534e2078350f40028cbddb69f0515f34 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1022.871484] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 534e2078350f40028cbddb69f0515f34 [ 1023.366622] env[62736]: DEBUG nova.network.neutron [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1023.367173] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 80f2c2a92fe146b189cf1bc285807314 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1023.374897] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 80f2c2a92fe146b189cf1bc285807314 [ 1023.869889] env[62736]: INFO nova.compute.manager [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: 14bc65e1-9305-4d75-8d27-d3077e7120c9] Took 1.02 seconds to deallocate network for instance. [ 1023.871711] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 76fd061dcb1f4f189bc10d9e6bd68092 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1023.902513] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 76fd061dcb1f4f189bc10d9e6bd68092 [ 1024.376898] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg b1725a78687f4d4cb9bdd1eb53aa23f5 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1024.407610] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b1725a78687f4d4cb9bdd1eb53aa23f5 [ 1024.900307] env[62736]: INFO nova.scheduler.client.report [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Deleted allocations for instance 14bc65e1-9305-4d75-8d27-d3077e7120c9 [ 1024.906789] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg a7bca800f91141e7ac3298c4d5fbdb84 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1024.924462] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a7bca800f91141e7ac3298c4d5fbdb84 [ 1025.408725] env[62736]: DEBUG oslo_concurrency.lockutils [None req-229ecb77-b262-4800-bb68-c7bd04213283 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "14bc65e1-9305-4d75-8d27-d3077e7120c9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.051s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1027.436832] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "abd9a353-18d2-45ac-842f-21c331cf96d8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1027.437143] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "abd9a353-18d2-45ac-842f-21c331cf96d8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1027.437572] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 8c55cc554e19414bb629297a9515d017 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1027.446143] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c55cc554e19414bb629297a9515d017 [ 1027.939288] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Starting instance... {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2432}} [ 1027.941173] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 4bbb56177ed6408896ec5724d5364861 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1027.972087] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bbb56177ed6408896ec5724d5364861 [ 1028.457890] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1028.458140] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1028.459584] env[62736]: INFO nova.compute.claims [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1028.461191] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg d0d551ef8a6e4a928fb7f61a97277507 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1028.491308] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0d551ef8a6e4a928fb7f61a97277507 [ 1028.964738] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 9a6a9b27543047c792f036a96e0851bb in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1028.974099] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9a6a9b27543047c792f036a96e0851bb [ 1029.492841] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-804a751e-c73c-4aff-99b5-164ce6ac9ba0 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.500240] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b64b6cbb-47f2-4195-b012-9f4127a82f10 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.529473] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-606a803f-2b48-465d-ae2f-1949e101a7b6 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.535948] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fde9aa86-f7ba-464c-9312-1f423263b9b7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.548309] env[62736]: DEBUG nova.compute.provider_tree [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1029.548780] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg c4a5534f8fc74491ac933bd41e0bb462 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1029.555709] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4a5534f8fc74491ac933bd41e0bb462 [ 1030.051079] env[62736]: DEBUG nova.scheduler.client.report [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1030.053415] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 1175c84c88bf40da8bca44f9d34394a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1030.064303] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1175c84c88bf40da8bca44f9d34394a3 [ 1030.555669] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 2.097s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1030.556223] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Start building networks asynchronously for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2829}} [ 1030.557978] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg ecc536ee3bb842b9b34b84aa9ea84855 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1030.587807] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ecc536ee3bb842b9b34b84aa9ea84855 [ 1031.062518] env[62736]: DEBUG nova.compute.utils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Using /dev/sd instead of None {{(pid=62736) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1031.063348] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 4289e232819344f2970f85d88967ad59 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1031.064474] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Allocating IP information in the background. {{(pid=62736) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1981}} [ 1031.064801] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] allocate_for_instance() {{(pid=62736) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1031.072859] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4289e232819344f2970f85d88967ad59 [ 1031.120130] env[62736]: DEBUG nova.policy [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88899e5c37af4416a1d5e65b61ca6218', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'af0fcc89cc8548f7b5f9689df14993c4', 'project_domain_id': 'default', 'roles': ['reader', 'member'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=62736) authorize /opt/stack/nova/nova/policy.py:203}} [ 1031.403026] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Successfully created port: d37d800f-d6c9-456c-95c7-f40794d52f37 {{(pid=62736) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1031.567766] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Start building block device mappings for instance. {{(pid=62736) _build_resources /opt/stack/nova/nova/compute/manager.py:2864}} [ 1031.570241] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 4b1bc2beed00448695c1ce8f60fb15ec in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1031.606952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b1bc2beed00448695c1ce8f60fb15ec [ 1032.076433] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 2452d783968c4e4ea2e264d319511176 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1032.117622] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2452d783968c4e4ea2e264d319511176 [ 1032.121991] env[62736]: DEBUG nova.compute.manager [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Received event network-changed-d37d800f-d6c9-456c-95c7-f40794d52f37 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1032.122435] env[62736]: DEBUG nova.compute.manager [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Refreshing instance network info cache due to event network-changed-d37d800f-d6c9-456c-95c7-f40794d52f37. {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11134}} [ 1032.122986] env[62736]: DEBUG oslo_concurrency.lockutils [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] Acquiring lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1032.123279] env[62736]: DEBUG oslo_concurrency.lockutils [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] Acquired lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1032.123585] env[62736]: DEBUG nova.network.neutron [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Refreshing network info cache for port d37d800f-d6c9-456c-95c7-f40794d52f37 {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1032.124209] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] Expecting reply to msg d297b6dc17a04e58a03bd191a463ac3f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1032.131046] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d297b6dc17a04e58a03bd191a463ac3f [ 1032.362721] env[62736]: ERROR nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Instance failed network setup after 1 attempt(s): nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1032.362721] env[62736]: ERROR nova.compute.manager Traceback (most recent call last): [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1032.362721] env[62736]: ERROR nova.compute.manager nwinfo = self.network_api.allocate_for_instance( [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1032.362721] env[62736]: ERROR nova.compute.manager created_port_ids = self._update_ports_for_instance( [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1032.362721] env[62736]: ERROR nova.compute.manager with excutils.save_and_reraise_exception(): [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1032.362721] env[62736]: ERROR nova.compute.manager self.force_reraise() [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1032.362721] env[62736]: ERROR nova.compute.manager raise self.value [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1032.362721] env[62736]: ERROR nova.compute.manager updated_port = self._update_port( [ 1032.362721] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1032.362721] env[62736]: ERROR nova.compute.manager _ensure_no_port_binding_failure(port) [ 1032.363244] env[62736]: ERROR nova.compute.manager File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1032.363244] env[62736]: ERROR nova.compute.manager raise exception.PortBindingFailed(port_id=port['id']) [ 1032.363244] env[62736]: ERROR nova.compute.manager nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1032.363244] env[62736]: ERROR nova.compute.manager [ 1032.363709] env[62736]: Traceback (most recent call last): [ 1032.363873] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/poll.py", line 111, in wait [ 1032.363873] env[62736]: listener.cb(fileno) [ 1032.363990] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1032.363990] env[62736]: result = function(*args, **kwargs) [ 1032.364108] env[62736]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1032.364108] env[62736]: return func(*args, **kwargs) [ 1032.364200] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1032.364200] env[62736]: raise e [ 1032.364293] env[62736]: File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1032.364293] env[62736]: nwinfo = self.network_api.allocate_for_instance( [ 1032.364386] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1032.364386] env[62736]: created_port_ids = self._update_ports_for_instance( [ 1032.364481] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1032.364481] env[62736]: with excutils.save_and_reraise_exception(): [ 1032.364572] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1032.364572] env[62736]: self.force_reraise() [ 1032.364659] env[62736]: File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1032.364659] env[62736]: raise self.value [ 1032.364752] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1032.364752] env[62736]: updated_port = self._update_port( [ 1032.364837] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1032.364837] env[62736]: _ensure_no_port_binding_failure(port) [ 1032.364928] env[62736]: File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1032.364928] env[62736]: raise exception.PortBindingFailed(port_id=port['id']) [ 1032.365023] env[62736]: nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1032.365129] env[62736]: Removing descriptor: 19 [ 1032.577792] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Start spawning the instance on the hypervisor. {{(pid=62736) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2638}} [ 1032.602325] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Getting desirable topologies for flavor Flavor(created_at=2024-08-19T12:29:16Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=192,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2024-08-19T12:28:59Z,direct_url=,disk_format='vmdk',id=81867c62-ef8e-483f-bfd2-854abdcd6db5,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='274176bd01e6438981fb4addec2b75f5',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2024-08-19T12:29:00Z,virtual_size=,visibility=), allow threads: False {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1032.602586] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Flavor limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1032.602743] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Image limits 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1032.602920] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Flavor pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1032.603059] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Image pref 0:0:0 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1032.603200] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=62736) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1032.603401] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1032.603557] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1032.603721] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Got 1 possible topologies {{(pid=62736) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1032.603879] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1032.604065] env[62736]: DEBUG nova.virt.hardware [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=62736) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1032.605075] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7869cd68-9d52-44de-b1ee-4c59f2d6f686 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.613286] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e784d99-f3e3-4890-ba75-b5521d94da83 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.627440] env[62736]: ERROR nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance failed to spawn: nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Traceback (most recent call last): [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 2894, in _build_resources [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] yield resources [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.driver.spawn(context, instance, image_meta, [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] vm_ref = self.build_virtual_machine(instance, [ 1032.627440] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] for vif in network_info: [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return self._sync_wrapper(fn, *args, **kwargs) [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.wait() [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self[:] = self._gt.wait() [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return self._exit_event.wait() [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1032.627835] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] current.throw(*self._exc) [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] result = function(*args, **kwargs) [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return func(*args, **kwargs) [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise e [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] nwinfo = self.network_api.allocate_for_instance( [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] created_port_ids = self._update_ports_for_instance( [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] with excutils.save_and_reraise_exception(): [ 1032.628227] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.force_reraise() [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise self.value [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] updated_port = self._update_port( [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] _ensure_no_port_binding_failure(port) [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise exception.PortBindingFailed(port_id=port['id']) [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1032.628654] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] [ 1032.628654] env[62736]: INFO nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Terminating instance [ 1032.631247] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1032.644641] env[62736]: DEBUG nova.network.neutron [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1032.712881] env[62736]: DEBUG nova.network.neutron [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1032.713481] env[62736]: INFO oslo_messaging._drivers.amqpdriver [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] Expecting reply to msg 53c189cc27ac4ce9a9d5b709d3aa26c0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1032.721403] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 53c189cc27ac4ce9a9d5b709d3aa26c0 [ 1033.215748] env[62736]: DEBUG oslo_concurrency.lockutils [req-5df98d95-e228-42e2-b68c-35ff5b2d9269 req-a0ab7f7e-2384-4a1b-90b6-757150af08d1 service nova] Releasing lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1033.216190] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquired lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1033.216376] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1033.216808] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 26a6e28f0c8f4ac18c65f6113ddd77f1 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1033.223402] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 26a6e28f0c8f4ac18c65f6113ddd77f1 [ 1033.733867] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1033.799420] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1033.799972] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 4c10970030b742848c39eaadb074e59f in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1033.807626] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4c10970030b742848c39eaadb074e59f [ 1034.143925] env[62736]: DEBUG nova.compute.manager [req-f8a4a1bd-ac89-4ba0-a283-a3c733ec9ade req-7241a769-6be2-4dc7-9e0a-c00d6ea989b7 service nova] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Received event network-vif-deleted-d37d800f-d6c9-456c-95c7-f40794d52f37 {{(pid=62736) external_instance_event /opt/stack/nova/nova/compute/manager.py:11129}} [ 1034.302122] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Releasing lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1034.302557] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Start destroying the instance on the hypervisor. {{(pid=62736) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3150}} [ 1034.302748] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Destroying instance {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1034.303026] env[62736]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-911c548a-b1fb-4bca-8a43-447e4d2fd8c3 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.312142] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa18d8e6-caea-4bcc-9acb-c42117e69de9 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.332659] env[62736]: WARNING nova.virt.vmwareapi.vmops [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance abd9a353-18d2-45ac-842f-21c331cf96d8 could not be found. [ 1034.332853] env[62736]: DEBUG nova.virt.vmwareapi.vmops [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance destroyed {{(pid=62736) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1034.333023] env[62736]: INFO nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Took 0.03 seconds to destroy the instance on the hypervisor. [ 1034.333245] env[62736]: DEBUG oslo.service.loopingcall [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=62736) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1034.333530] env[62736]: DEBUG nova.compute.manager [-] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1034.333630] env[62736]: DEBUG nova.network.neutron [-] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1034.347203] env[62736]: DEBUG nova.network.neutron [-] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1034.347606] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg a82b24ab3ec5469aa6e9d1a537cbf34c in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1034.355189] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a82b24ab3ec5469aa6e9d1a537cbf34c [ 1034.850023] env[62736]: DEBUG nova.network.neutron [-] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1034.850482] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Expecting reply to msg 3aaf5dcfcadc425d8235ce65ff4ef5a3 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1034.857952] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3aaf5dcfcadc425d8235ce65ff4ef5a3 [ 1035.353296] env[62736]: INFO nova.compute.manager [-] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Took 1.02 seconds to deallocate network for instance. [ 1035.355868] env[62736]: DEBUG nova.compute.claims [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Aborting claim: {{(pid=62736) abort /opt/stack/nova/nova/compute/claims.py:84}} [ 1035.356066] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1035.356291] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1035.358233] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg e6016ac322b94709a4fc4700250357b4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1035.390156] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6016ac322b94709a4fc4700250357b4 [ 1035.889890] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a53ac8c8-73d9-4cb1-994c-8b7d5cba340f {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1035.897120] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-435602bf-a65b-4759-978d-8e1a47744b07 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1035.927015] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd1ed049-73c7-465d-9774-f9f8b3de560d {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1035.933473] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b2796f9-16cf-4b3e-951f-93cbcc5402a2 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1035.945838] env[62736]: DEBUG nova.compute.provider_tree [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1035.946315] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg f226f36fc12e40b484b0188902b1140e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1035.952927] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f226f36fc12e40b484b0188902b1140e [ 1036.448910] env[62736]: DEBUG nova.scheduler.client.report [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1036.452269] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg b765e238e8ed45fc83e4385a6ad1af01 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1036.464494] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b765e238e8ed45fc83e4385a6ad1af01 [ 1036.954490] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.598s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1036.955128] env[62736]: ERROR nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Failed to build and run instance: nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Traceback (most recent call last): [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 2641, in _build_and_run_instance [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.driver.spawn(context, instance, image_meta, [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] vm_ref = self.build_virtual_machine(instance, [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] vif_infos = vmwarevif.get_vif_info(self._session, [ 1036.955128] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] for vif in network_info: [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__ [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return self._sync_wrapper(fn, *args, **kwargs) [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.wait() [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/model.py", line 635, in wait [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self[:] = self._gt.wait() [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return self._exit_event.wait() [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] current.throw(*self._exc) [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main [ 1036.955484] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] result = function(*args, **kwargs) [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] return func(*args, **kwargs) [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 2011, in _allocate_network_async [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise e [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/compute/manager.py", line 1989, in _allocate_network_async [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] nwinfo = self.network_api.allocate_for_instance( [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] created_port_ids = self._update_ports_for_instance( [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] with excutils.save_and_reraise_exception(): [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] self.force_reraise() [ 1036.955900] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise self.value [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] updated_port = self._update_port( [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] _ensure_no_port_binding_failure(port) [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] raise exception.PortBindingFailed(port_id=port['id']) [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] nova.exception.PortBindingFailed: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. [ 1036.956305] env[62736]: ERROR nova.compute.manager [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] [ 1036.956305] env[62736]: DEBUG nova.compute.utils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. {{(pid=62736) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1036.957578] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Build of instance abd9a353-18d2-45ac-842f-21c331cf96d8 was re-scheduled: Binding failed for port d37d800f-d6c9-456c-95c7-f40794d52f37, please check neutron logs for more information. {{(pid=62736) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2480}} [ 1036.957972] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Unplugging VIFs for instance {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3006}} [ 1036.958200] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquiring lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1036.958345] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Acquired lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1036.958502] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Building network info cache for instance {{(pid=62736) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1036.958903] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 2cc2f34bf1a14c7ab081fcd81d1b7f03 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1036.965272] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2cc2f34bf1a14c7ab081fcd81d1b7f03 [ 1037.474553] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1037.541369] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1037.541909] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg bc951da2acdb4512913718b323a8767e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1037.549413] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc951da2acdb4512913718b323a8767e [ 1038.044058] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Releasing lock "refresh_cache-abd9a353-18d2-45ac-842f-21c331cf96d8" {{(pid=62736) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1038.044416] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=62736) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3029}} [ 1038.044543] env[62736]: DEBUG nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Deallocating network for instance {{(pid=62736) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2289}} [ 1038.044676] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] deallocate_for_instance() {{(pid=62736) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1038.059086] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Instance cache missing network info. {{(pid=62736) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1038.059667] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 81f887740c784894911f4cbd079c8f55 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1038.066264] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 81f887740c784894911f4cbd079c8f55 [ 1038.562562] env[62736]: DEBUG nova.network.neutron [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Updating instance_info_cache with network_info: [] {{(pid=62736) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1038.563077] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 66a68b318434438cbb05f9434056c0a0 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1038.572022] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 66a68b318434438cbb05f9434056c0a0 [ 1039.065969] env[62736]: INFO nova.compute.manager [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] [instance: abd9a353-18d2-45ac-842f-21c331cf96d8] Took 1.02 seconds to deallocate network for instance. [ 1039.067861] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 487cbbec81f643438f58527693a22eb9 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1039.098208] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 487cbbec81f643438f58527693a22eb9 [ 1039.572946] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg 2973b04aeb404db4a41ae9870051f190 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1039.604473] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2973b04aeb404db4a41ae9870051f190 [ 1040.094309] env[62736]: INFO nova.scheduler.client.report [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Deleted allocations for instance abd9a353-18d2-45ac-842f-21c331cf96d8 [ 1040.101712] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Expecting reply to msg b2d20833e7174461a0aead9e3a191e64 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1040.117344] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b2d20833e7174461a0aead9e3a191e64 [ 1040.603744] env[62736]: DEBUG oslo_concurrency.lockutils [None req-59e9656d-9cff-4e73-b73c-886ad4974d48 tempest-AttachVolumeShelveTestJSON-1951917446 tempest-AttachVolumeShelveTestJSON-1951917446-project-member] Lock "abd9a353-18d2-45ac-842f-21c331cf96d8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 13.166s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1041.136800] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.137135] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.137176] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Starting heal instance info cache {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9926}} [ 1041.137297] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Rebuilding the list of instances to heal {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9930}} [ 1041.137947] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg e85469e2ee5e49bfa51815211e27a1c4 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1041.146809] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e85469e2ee5e49bfa51815211e27a1c4 [ 1041.640314] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Didn't find any instances for network info cache update. {{(pid=62736) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10012}} [ 1041.640596] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.640763] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.640915] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.641064] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.641208] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.641353] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.641484] env[62736]: DEBUG nova.compute.manager [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=62736) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10545}} [ 1041.641627] env[62736]: DEBUG oslo_service.periodic_task [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Running periodic task ComputeManager.update_available_resource {{(pid=62736) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1041.642003] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 79da40c227af42e6a1c1c13940d6f069 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1041.651707] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 79da40c227af42e6a1c1c13940d6f069 [ 1042.150555] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1042.150555] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1042.150555] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1042.150555] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=62736) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1042.150555] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58b02861-2721-4330-b23d-4b29f7700f30 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.154630] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42440790-e659-4408-bdb5-6d6c2274cc6a {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.170207] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1e4b4810-5f43-4802-968a-8470c46d289e {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.178886] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-949d2bfe-21cf-4710-a22e-6e3bb6f55cbb {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1042.205387] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181745MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=62736) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1042.205549] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1042.205753] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1042.206596] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 3f96479fe0bb4174ab1a2a326e189a74 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1042.215671] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3f96479fe0bb4174ab1a2a326e189a74 [ 1042.708755] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 9b91989de13642b6aefd65a5f5f26a83 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1042.718989] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9b91989de13642b6aefd65a5f5f26a83 [ 1043.225268] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1043.225516] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=62736) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1043.238695] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-73d77636-6259-4456-8fbd-0fe40820acf4 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.246474] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8abc8b5-b1dd-407e-a938-6ff2a1817b73 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.279055] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e40f8602-d419-4bc3-84e5-ac6d90adef60 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.286273] env[62736]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fb8fb95-3901-488d-9e3b-e790c46e61d7 {{(pid=62736) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1043.298953] env[62736]: DEBUG nova.compute.provider_tree [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed in ProviderTree for provider: 0c9afe22-9d34-458c-8118-58661faecbae {{(pid=62736) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1043.299536] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 4bbf6bf37206403fa6d3e519cc8b166e in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1043.308868] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4bbf6bf37206403fa6d3e519cc8b166e [ 1043.802608] env[62736]: DEBUG nova.scheduler.client.report [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Inventory has not changed for provider 0c9afe22-9d34-458c-8118-58661faecbae based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=62736) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:954}} [ 1043.804760] env[62736]: INFO oslo_messaging._drivers.amqpdriver [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Expecting reply to msg 2df3a4c6d7db4be782ce33f9e75afe06 in queue reply_c579f1f3113446edbaca3803c2d26d58 [ 1043.815922] env[62736]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2df3a4c6d7db4be782ce33f9e75afe06 [ 1044.307503] env[62736]: DEBUG nova.compute.resource_tracker [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=62736) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1044.307891] env[62736]: DEBUG oslo_concurrency.lockutils [None req-a7db153b-e58a-4a35-9a6c-905fd786100d None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 2.102s {{(pid=62736) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}}